diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..25df0e61e57852a0eabfd57c1818d6cb215f029e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/run-8ce8thqh.wandb filter=lfs diff=lfs merge=lfs -text +lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/run-pkxhwzf2.wandb filter=lfs diff=lfs merge=lfs -text +lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/run-umsnhs3c.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/act_so101_test/wandb/debug-internal.log b/act_so101_test/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cfa6e22a13948b82b0cc19ee80d60d6d1c313433 --- /dev/null +++ b/act_so101_test/wandb/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-12-30T22:17:50.001599125+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-30T22:17:50.459815178+05:30","level":"INFO","msg":"stream: created new stream","id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459832404+05:30","level":"INFO","msg":"stream: started","id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459873772+05:30","level":"INFO","msg":"writer: started","stream_id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459884981+05:30","level":"INFO","msg":"handler: started","stream_id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459918165+05:30","level":"INFO","msg":"sender: started","stream_id":"q1o5aaw1"} +{"time":"2025-12-30T22:18:28.195147193+05:30","level":"INFO","msg":"stream: closing","id":"q1o5aaw1"} diff --git a/act_so101_test/wandb/debug.log b/act_so101_test/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f6b224cc479fe6b948fca9c372df57dd147ea18f --- /dev/null +++ b/act_so101_test/wandb/debug.log @@ -0,0 +1,56 @@ +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Configure stats pid to 2365002 +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug.log +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-internal.log +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:init():813] calling init triggers +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'lerobot-act', 'root': '/home/zeux/lerobot_shadow/lerobotact-dataset', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'local/act-so101-model', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'chunk_size': 100, 'n_action_steps': 100, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/act_so101_test', 'job_name': 'act', 'resume': False, 'seed': 1000, 'num_workers': 1, 'batch_size': 8, 'steps': 20000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:init():854] starting backend +2025-12-30 22:17:49,996 INFO MainThread:2365002 [wandb_init.py:init():857] sending inform_init request +2025-12-30 22:17:49,999 INFO MainThread:2365002 [wandb_init.py:init():865] backend started and connected +2025-12-30 22:17:50,000 INFO MainThread:2365002 [wandb_init.py:init():936] updated telemetry +2025-12-30 22:17:50,000 INFO MainThread:2365002 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-30 22:17:50,852 INFO MainThread:2365002 [wandb_init.py:init():1011] starting run threads in backend +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_console_start():2506] atexit reg +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-30 22:18:28,195 INFO wandb-AsyncioManager-main:2365002 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-30 22:18:28,195 INFO wandb-AsyncioManager-main:2365002 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles. +2025-12-30 22:18:28,592 ERROR wandb-AsyncioManager-main:2365002 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions + await fn() + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 65, in _send_server_request + await self._writer.drain() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 371, in drain + await self._protocol._drain_helper() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 167, in _drain_helper + raise ConnectionResetError('Connection lost') +ConnectionResetError: Connection lost +2025-12-30 22:18:28,601 ERROR wandb-AsyncioManager-main:2365002 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions + await fn() + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 65, in _send_server_request + await self._writer.drain() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 371, in drain + await self._protocol._drain_helper() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 167, in _drain_helper + raise ConnectionResetError('Connection lost') +ConnectionResetError: Connection lost +2025-12-30 22:18:28,601 ERROR wandb-AsyncioManager-main:2365002 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions + await fn() + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 65, in _send_server_request + await self._writer.drain() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 371, in drain + await self._protocol._drain_helper() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 167, in _drain_helper + raise ConnectionResetError('Connection lost') +ConnectionResetError: Connection lost diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/config.yaml b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..618cdf147fb2125573a09c56ca9464f9a2928f29 --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/config.yaml @@ -0,0 +1,225 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + jldifxu3qy62ax4zkqe4sl8faxnhsjg9: + args: + - --dataset.repo_id=lerobot-act + - --dataset.root=/home/zeux/lerobot_shadow/lerobotact-dataset + - --dataset.video_backend=pyav + - --policy.type=act + - --policy.repo_id=local/act-so101-model + - --output_dir=outputs/train/act_so101_test + - --policy.device=cuda + - --policy.use_amp=true + - --batch_size=8 + - --num_workers=1 + - --steps=20000 + - --wandb.enable=true + codePath: miniconda3/envs/lerobot/bin/lerobot-train + codePathLocal: miniconda3/envs/lerobot/bin/lerobot-train + cpu_count: 24 + cpu_count_logical: 24 + cudaVersion: "13.0" + disk: + /: + total: "200449576960" + used: "34402668544" + executable: /home/zeux/miniconda3/envs/lerobot/bin/python3.10 + gpu: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + gpu_count: 1 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + uuid: GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e + host: zeux + memory: + total: "134489886720" + os: Linux-6.8.0-90-generic-x86_64-with-glibc2.35 + program: /home/zeux/miniconda3/envs/lerobot/bin/lerobot-train + python: CPython 3.10.19 + root: outputs/train/act_so101_test + startedAt: "2025-12-30T16:47:49.793724Z" + writerId: jldifxu3qy62ax4zkqe4sl8faxnhsjg9 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "3": + - 13 + - 15 + - 16 + "4": 3.10.19 + "5": 0.21.4 + "6": 4.57.3 + "10": + - 21 + "12": 0.21.4 + "13": linux-x86_64 +batch_size: + value: 8 +checkpoint_path: + value: null +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: lerobot-act + revision: null + root: /home/zeux/lerobot_shadow/lerobotact-dataset + streaming: false + use_imagenet_stats: true + video_backend: pyav +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: act +log_freq: + value: 200 +num_workers: + value: 1 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 1e-05 + type: adamw + weight_decay: 0.0001 +output_dir: + value: outputs/train/act_so101_test +policy: + value: + chunk_size: 100 + device: cuda + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + kl_weight: 10 + latent_dim: 32 + license: null + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: MEAN_STD + optimizer_lr: 1e-05 + optimizer_lr_backbone: 1e-05 + optimizer_weight_decay: 0.0001 + pre_norm: false + pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1 + pretrained_path: null + private: null + push_to_hub: true + replace_final_stride_with_dilation: false + repo_id: local/act-so101-model + tags: null + temporal_ensemble_coeff: null + type: act + use_amp: true + use_vae: true + vision_backbone: resnet18 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 20000 +scheduler: + value: null +seed: + value: 1000 +steps: + value: 20000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/output.log b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d0028a1f92d3cc6d1e4aafc0e7120c0a8f76030b --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/output.log @@ -0,0 +1,53 @@ +INFO 2025-12-30 22:17:50 db_utils.py:102 Logs will be synced with wandb. +INFO 2025-12-30 22:17:50 db_utils.py:103 Track this run --> https://wandb.ai/nwaves-ai-nwaves/lerobot/runs/q1o5aaw1 +INFO 2025-12-30 22:17:50 ot_train.py:183 Creating dataset +INFO 2025-12-30 22:17:50 ot_train.py:202 Creating policy +INFO 2025-12-30 22:17:51 ot_train.py:247 Creating optimizer and scheduler +INFO 2025-12-30 22:17:51 ot_train.py:259 Output dir: outputs/train/act_so101_test +INFO 2025-12-30 22:17:51 ot_train.py:264 cfg.steps=20000 (20K) +INFO 2025-12-30 22:17:51 ot_train.py:265 dataset.num_frames=17774 (18K) +INFO 2025-12-30 22:17:51 ot_train.py:266 dataset.num_episodes=20 +INFO 2025-12-30 22:17:51 ot_train.py:269 Effective batch size: 8 x 1 = 8 +INFO 2025-12-30 22:17:51 ot_train.py:270 num_learnable_params=51597190 (52M) +INFO 2025-12-30 22:17:51 ot_train.py:271 num_total_params=51597190 (52M) +INFO 2025-12-30 22:17:51 ot_train.py:327 Start offline training on a fixed dataset +Traceback (most recent call last): + File "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", line 7, in + sys.exit(main()) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 449, in main + train() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/configs/parser.py", line 233, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 335, in train + train_tracker, output_dict = update_policy( + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 95, in update_policy + accelerator.backward(loss) + File "/home/zeux/.local/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward + loss.backward(**kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward + torch.autograd.backward( + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward + _engine_run_backward( + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt +Traceback (most recent call last): + File "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", line 7, in + sys.exit(main()) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 449, in main + train() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/configs/parser.py", line 233, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 335, in train + train_tracker, output_dict = update_policy( + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 95, in update_policy + accelerator.backward(loss) + File "/home/zeux/.local/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward + loss.backward(**kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward + torch.autograd.backward( + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward + _engine_run_backward( + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/requirements.txt b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..24e53687369669d9994213578166cd6f5de74e67 --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/requirements.txt @@ -0,0 +1,237 @@ +nvidia-cusolver==12.0.3.29 +nvidia-curand==10.4.0.35 +anyio==4.12.0 +yarl==1.22.0 +comfyui-workflow-templates-media-other==0.3.15 +tomlkit==0.13.3 +cloudpickle==3.1.2 +typing_extensions==4.15.0 +sentry-sdk==2.48.0 +semver==3.0.4 +mpmath==1.3.0 +draccus==0.10.0 +multidict==6.7.0 +nvidia-cufile-cu12==1.11.1.6 +tokenizers==0.22.1 +GitPython==3.1.45 +datasets==4.1.1 +PySide6_Essentials==6.10.1 +comfyui-workflow-templates-media-image==0.3.17 +Pygments==2.19.2 +aiohappyeyeballs==2.6.1 +protobuf==6.33.2 +PySide6==6.10.1 +comfyui-workflow-templates-core==0.3.12 +PySide2==5.15.2.1 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +scipy==1.15.3 +frozenlist==1.8.0 +nvidia-nvshmem-cu13==3.3.24 +transformers==4.57.3 +typing-inspection==0.4.2 +einops==0.8.1 +shellingham==1.5.4 +Farama-Notifications==0.0.4 +psutil==7.1.3 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +nvidia-cufft==12.0.0.15 +cookiecutter==2.6.0 +rich==14.2.0 +SQLAlchemy==2.0.44 +pydantic_core==2.41.5 +nvidia-nvjitlink==13.0.39 +questionary==2.1.1 +requests==2.32.5 +nvidia-cuda-nvrtc==13.0.48 +torchcodec==0.5 +PyOpenGL-accelerate==3.1.10 +comfyui_frontend_package==1.33.10 +networkx==3.4.2 +nvidia-nvtx==13.0.39 +setuptools==80.9.0 +trampoline==0.1.2 +pandas==2.3.3 +httpx==0.28.1 +av==15.1.0 +comfy-cli==1.5.3 +PyOpenGL==3.1.10 +tomli==2.3.0 +safetensors==0.7.0 +regex==2025.11.3 +wcwidth==0.2.14 +pyyaml-include==1.4.1 +gymnasium==1.2.3 +kornia==0.8.2 +packaging==25.0 +nvidia-nccl-cu13==2.27.7 +pydantic==2.12.5 +mixpanel==5.0.0 +markdown-it-py==4.0.0 +multiprocess==0.70.16 +usd-exchange==2.1.0 +h11==0.16.0 +mergedeep==1.3.4 +shiboken6==6.10.1 +text-unidecode==1.3 +deepdiff==8.6.1 +toml==0.10.2 +aiosignal==1.4.0 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +uv==0.9.16 +jsonlines==4.0.0 +xxhash==3.6.0 +numpy==2.2.6 +nvidia-cuda-runtime==13.0.48 +arrow==1.4.0 +spandrel==0.4.1 +nvidia-cuda-cupti==13.0.48 +python-dotenv==1.2.1 +async-timeout==5.0.1 +urllib3==2.6.2 +fsspec==2025.9.0 +nvidia-cusparse==12.6.2.49 +binaryornot==0.4.4 +nvidia-cufile==1.15.0.42 +mdurl==0.1.2 +pydantic-settings==2.12.0 +gitdb==4.0.12 +kornia_rs==0.1.10 +attrs==25.4.0 +mypy_extensions==1.1.0 +tqdm==4.67.1 +alembic==1.17.2 +comfyui-embedded-docs==0.3.1 +python-slugify==8.0.4 +cmake==4.1.3 +smmap==5.0.2 +comfyui-workflow-templates-media-api==0.3.14 +torchsde==0.2.6 +pathspec==0.12.1 +sentencepiece==0.2.1 +charset-normalizer==3.4.4 +nvidia-cusparselt-cu13==0.8.0 +python-xlib==0.33 +ruff==0.14.8 +platformdirs==4.5.1 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-cublas==13.0.0.19 +prompt_toolkit==3.0.52 +ImageIO==2.37.2 +termcolor==3.3.0 +wandb==0.21.4 +annotated-types==0.7.0 +comfyui-workflow-templates-media-video==0.3.12 +typer==0.20.0 +dill==0.4.0 +propcache==0.4.1 +typing-inspect==0.9.0 +greenlet==3.3.0 +hf-xet==1.2.0 +shiboken2==5.15.2.1 +tzdata==2025.2 +aiohttp==3.13.2 +websocket-client==1.9.0 +nvidia-cudnn-cu13==9.13.0.50 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +PySide6_Addons==6.10.1 +asgiref==3.11.0 +httpcore==1.0.9 +exceptiongroup==1.3.1 +filelock==3.20.0 +comfyui_workflow_templates==0.7.25 +opencv-python-headless==4.12.0.88 +nvidia-curand-cu12==10.3.7.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-nvtx-cu12==12.6.77 +cloudpickle==3.1.2 +pytz==2025.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-cusparselt-cu12==0.7.1 +sentry-sdk==2.48.0 +importlib_metadata==8.7.1 +draccus==0.10.0 +nvidia-cufile-cu12==1.11.1.6 +datasets==4.1.1 +nvidia-cusolver-cu12==11.7.1.2 +protobuf==6.33.2 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +setuptools==80.9.0 +nvidia-cuda-cupti-cu12==12.6.80 +Farama-Notifications==0.0.4 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +torchvision==0.24.1+cu130 +requests==2.32.5 +torchcodec==0.5 +idna==3.11 +zipp==3.23.0 +pandas==2.3.3 +av==15.1.0 +nvidia-cusparse-cu12==12.5.4.2 +pyyaml-include==1.4.1 +gymnasium==1.2.3 +nvidia-cuda-runtime-cu12==12.6.77 +triton==3.6.0+git9844da95 +triton==3.5.1 +multiprocess==0.70.16 +lerobot==0.4.2 +Jinja2==3.1.6 +click==8.3.1 +nvidia-nccl-cu12==2.28.9 +nvidia-nccl-cu12==2.26.2 +mergedeep==1.3.4 +deepdiff==8.6.1 +toml==0.10.2 +sympy==1.14.0 +hf_transfer==0.1.9 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +jsonlines==4.0.0 +xxhash==3.6.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +urllib3==2.6.2 +fsspec==2025.9.0 +pillow==12.0.0 +pytorch-triton==3.1.0+cf34004b8a +cuda-pathfinder==1.2.2 +mypy_extensions==1.1.0 +cmake==4.1.3 +certifi==2025.11.12 +num2words==0.5.14 +python-xlib==0.33 +nvidia-nvjitlink-cu12==12.6.85 +platformdirs==4.5.1 +torch==2.9.1+cu130 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-nvshmem-cu12==3.4.5 +ImageIO==2.37.2 +termcolor==3.2.0 +six==1.17.0 +wandb==0.21.4 +MarkupSafe==3.0.3 +wheel==0.45.1 +dill==0.4.0 +typing-inspect==0.9.0 +cuda-bindings==12.9.4 +pip==25.3 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +nvidia-cublas-cu12==12.6.4.1 +docopt==0.6.2 +nvidia-cufft-cu12==11.3.0.4 +opencv-python-headless==4.12.0.88 diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/wandb-metadata.json b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8531d992714a903d82917c19d196e02805c34de2 --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/wandb-metadata.json @@ -0,0 +1,49 @@ +{ + "os": "Linux-6.8.0-90-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2025-12-30T16:47:49.793724Z", + "args": [ + "--dataset.repo_id=lerobot-act", + "--dataset.root=/home/zeux/lerobot_shadow/lerobotact-dataset", + "--dataset.video_backend=pyav", + "--policy.type=act", + "--policy.repo_id=local/act-so101-model", + "--output_dir=outputs/train/act_so101_test", + "--policy.device=cuda", + "--policy.use_amp=true", + "--batch_size=8", + "--num_workers=1", + "--steps=20000", + "--wandb.enable=true" + ], + "program": "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", + "codePath": "miniconda3/envs/lerobot/bin/lerobot-train", + "codePathLocal": "miniconda3/envs/lerobot/bin/lerobot-train", + "root": "outputs/train/act_so101_test", + "host": "zeux", + "executable": "/home/zeux/miniconda3/envs/lerobot/bin/python3.10", + "cpu_count": 24, + "cpu_count_logical": 24, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "gpu_count": 1, + "disk": { + "/": { + "total": "200449576960", + "used": "34402668544" + } + }, + "memory": { + "total": "134489886720" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e" + } + ], + "cudaVersion": "13.0", + "writerId": "jldifxu3qy62ax4zkqe4sl8faxnhsjg9" +} \ No newline at end of file diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/wandb-summary.json b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3e4dda75148b11087b7ca4d383b6747633b42a57 --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":37},"_runtime":37} \ No newline at end of file diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-core.log b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1271780325f72f8e85c32e4421ba2fb33a1d200e --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2025-12-30T22:17:49.803961932+05:30","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpp_mdrrv8/port-2365002.txt","pid":2365002,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-12-30T22:17:49.804217382+05:30","level":"INFO","msg":"server: will exit if parent process dies","ppid":2365002} +{"time":"2025-12-30T22:17:49.804200654+05:30","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2365002-2365059-351286409/socket","Net":"unix"}} +{"time":"2025-12-30T22:17:49.996879889+05:30","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-12-30T22:17:50.001521224+05:30","level":"INFO","msg":"handleInformInit: received","streamId":"q1o5aaw1","id":"1(@)"} +{"time":"2025-12-30T22:17:50.459834724+05:30","level":"INFO","msg":"handleInformInit: stream started","streamId":"q1o5aaw1","id":"1(@)"} +{"time":"2025-12-30T22:18:28.195121336+05:30","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-12-30T22:18:28.195172479+05:30","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-12-30T22:18:28.195193073+05:30","level":"INFO","msg":"server is shutting down"} +{"time":"2025-12-30T22:18:28.195214713+05:30","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-12-30T22:18:28.195230222+05:30","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2365002-2365059-351286409/socket","Net":"unix"}} +{"time":"2025-12-30T22:18:28.922840586+05:30","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-internal.log b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cfa6e22a13948b82b0cc19ee80d60d6d1c313433 --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-12-30T22:17:50.001599125+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-30T22:17:50.459815178+05:30","level":"INFO","msg":"stream: created new stream","id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459832404+05:30","level":"INFO","msg":"stream: started","id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459873772+05:30","level":"INFO","msg":"writer: started","stream_id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459884981+05:30","level":"INFO","msg":"handler: started","stream_id":"q1o5aaw1"} +{"time":"2025-12-30T22:17:50.459918165+05:30","level":"INFO","msg":"sender: started","stream_id":"q1o5aaw1"} +{"time":"2025-12-30T22:18:28.195147193+05:30","level":"INFO","msg":"stream: closing","id":"q1o5aaw1"} diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug.log b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f6b224cc479fe6b948fca9c372df57dd147ea18f --- /dev/null +++ b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug.log @@ -0,0 +1,56 @@ +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Configure stats pid to 2365002 +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug.log +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/logs/debug-internal.log +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:init():813] calling init triggers +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'lerobot-act', 'root': '/home/zeux/lerobot_shadow/lerobotact-dataset', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'local/act-so101-model', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'chunk_size': 100, 'n_action_steps': 100, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/act_so101_test', 'job_name': 'act', 'resume': False, 'seed': 1000, 'num_workers': 1, 'batch_size': 8, 'steps': 20000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-30 22:17:49,794 INFO MainThread:2365002 [wandb_init.py:init():854] starting backend +2025-12-30 22:17:49,996 INFO MainThread:2365002 [wandb_init.py:init():857] sending inform_init request +2025-12-30 22:17:49,999 INFO MainThread:2365002 [wandb_init.py:init():865] backend started and connected +2025-12-30 22:17:50,000 INFO MainThread:2365002 [wandb_init.py:init():936] updated telemetry +2025-12-30 22:17:50,000 INFO MainThread:2365002 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-30 22:17:50,852 INFO MainThread:2365002 [wandb_init.py:init():1011] starting run threads in backend +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_console_start():2506] atexit reg +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-30 22:17:50,911 INFO MainThread:2365002 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-30 22:18:28,195 INFO wandb-AsyncioManager-main:2365002 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-30 22:18:28,195 INFO wandb-AsyncioManager-main:2365002 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles. +2025-12-30 22:18:28,592 ERROR wandb-AsyncioManager-main:2365002 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions + await fn() + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 65, in _send_server_request + await self._writer.drain() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 371, in drain + await self._protocol._drain_helper() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 167, in _drain_helper + raise ConnectionResetError('Connection lost') +ConnectionResetError: Connection lost +2025-12-30 22:18:28,601 ERROR wandb-AsyncioManager-main:2365002 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions + await fn() + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 65, in _send_server_request + await self._writer.drain() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 371, in drain + await self._protocol._drain_helper() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 167, in _drain_helper + raise ConnectionResetError('Connection lost') +ConnectionResetError: Connection lost +2025-12-30 22:18:28,601 ERROR wandb-AsyncioManager-main:2365002 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions + await fn() + File "/home/zeux/.local/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 65, in _send_server_request + await self._writer.drain() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 371, in drain + await self._protocol._drain_helper() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/asyncio/streams.py", line 167, in _drain_helper + raise ConnectionResetError('Connection lost') +ConnectionResetError: Connection lost diff --git a/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/run-q1o5aaw1.wandb b/act_so101_test/wandb/run-20251230_221749-q1o5aaw1/run-q1o5aaw1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lerobot-act_so101_test/checkpoints/020000/pretrained_model/config.json b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a015180999136bfcd61fc3215f1fdda7b347e650 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/config.json @@ -0,0 +1,70 @@ +{ + "type": "act", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.side": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "local/lerobot-act-so101-model", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 100, + "n_action_steps": 100, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/020000/pretrained_model/model.safetensors b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a7c6e78ed4f36dcfe925196dddfdb60fd15bfbde --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0173494afd88d26d228b0454fafb43979977942d75c9a380f4211e9f5c641322 +size 206699736 diff --git a/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor.json b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..9ccb342474cecbf2bdbb034d27169a351ad96d2a --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d3609578476a8f3edf2a49807eaa304f9bc9f0d2 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a892bb501c57ca790c3bd1425b3f9c22828f516040443efdee18efb5894a8b +size 7520 diff --git a/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor.json b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..d56833d4e0ef1bce1add1bb45f6b17449f9c36da --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,64 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.side": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d3609578476a8f3edf2a49807eaa304f9bc9f0d2 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a892bb501c57ca790c3bd1425b3f9c22828f516040443efdee18efb5894a8b +size 7520 diff --git a/lerobot-act_so101_test/checkpoints/020000/pretrained_model/train_config.json b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfdb1003e375d196f070305084c8832d42ece913 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/pretrained_model/train_config.json @@ -0,0 +1,193 @@ +{ + "dataset": { + "repo_id": "lerobot-act", + "root": "/home/zeux/lerobot-act-dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav", + "streaming": false + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.side": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "local/lerobot-act-so101-model", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 100, + "n_action_steps": 100, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/lerobot-act_so101_test", + "job_name": "act", + "resume": false, + "seed": 1000, + "num_workers": 8, + "batch_size": 64, + "steps": 35000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "8ce8thqh", + "mode": null + }, + "checkpoint_path": null, + "rename_map": {} +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/020000/training_state/optimizer_param_groups.json b/lerobot-act_so101_test/checkpoints/020000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/020000/training_state/optimizer_state.safetensors b/lerobot-act_so101_test/checkpoints/020000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4375a149f6e64fdd9ac4223a76b375fd057a2abd --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c25e18d5aab89b1b6c01e2844f144f3e39f4b2df1b3584073e5c0618bd517f2f +size 412817652 diff --git a/lerobot-act_so101_test/checkpoints/020000/training_state/rng_state.safetensors b/lerobot-act_so101_test/checkpoints/020000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5d1860e062968a72c43f814cdd7d8d68f37594fb --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08684fe5df9e737d72881edf41caa4456fc8fa51b57ee7ef8389f6e7b1b0af67 +size 15708 diff --git a/lerobot-act_so101_test/checkpoints/020000/training_state/training_step.json b/lerobot-act_so101_test/checkpoints/020000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9bb47026c5d5237ca6fc5dbff6020dd122ea05 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/020000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 20000 +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/035000/pretrained_model/config.json b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a015180999136bfcd61fc3215f1fdda7b347e650 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/config.json @@ -0,0 +1,70 @@ +{ + "type": "act", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.side": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "local/lerobot-act-so101-model", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 100, + "n_action_steps": 100, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/035000/pretrained_model/model.safetensors b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..29dfd210c95fa34c9875d2c4c9aa3dbf37a45e44 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05305bec1095fe152694317984d4623935dcf9a4b8c0bbea3fa5244c02af351a +size 206699736 diff --git a/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_postprocessor.json b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..9ccb342474cecbf2bdbb034d27169a351ad96d2a --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d3609578476a8f3edf2a49807eaa304f9bc9f0d2 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a892bb501c57ca790c3bd1425b3f9c22828f516040443efdee18efb5894a8b +size 7520 diff --git a/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_preprocessor.json b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..d56833d4e0ef1bce1add1bb45f6b17449f9c36da --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,64 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.side": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d3609578476a8f3edf2a49807eaa304f9bc9f0d2 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a892bb501c57ca790c3bd1425b3f9c22828f516040443efdee18efb5894a8b +size 7520 diff --git a/lerobot-act_so101_test/checkpoints/035000/pretrained_model/train_config.json b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dfdb1003e375d196f070305084c8832d42ece913 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/pretrained_model/train_config.json @@ -0,0 +1,193 @@ +{ + "dataset": { + "repo_id": "lerobot-act", + "root": "/home/zeux/lerobot-act-dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav", + "streaming": false + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.side": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "local/lerobot-act-so101-model", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 100, + "n_action_steps": 100, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/lerobot-act_so101_test", + "job_name": "act", + "resume": false, + "seed": 1000, + "num_workers": 8, + "batch_size": 64, + "steps": 35000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "8ce8thqh", + "mode": null + }, + "checkpoint_path": null, + "rename_map": {} +} \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/035000/training_state/optimizer_param_groups.json b/lerobot-act_so101_test/checkpoints/035000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/lerobot-act_so101_test/checkpoints/035000/training_state/optimizer_state.safetensors b/lerobot-act_so101_test/checkpoints/035000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..af1d83593219299547a4f4f087d48ff3f08ea14a --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b07539463c39fa59f0c98b000c889050575e06d91b79471d458d836a36e7e71 +size 412817652 diff --git a/lerobot-act_so101_test/checkpoints/035000/training_state/rng_state.safetensors b/lerobot-act_so101_test/checkpoints/035000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..59934024723cf25d002c8aa645fa93a03354b62c --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb2fd6abc98e0392a9d887f83cba768640011f91bf678d7c01a644c40ebae61a +size 15708 diff --git a/lerobot-act_so101_test/checkpoints/035000/training_state/training_step.json b/lerobot-act_so101_test/checkpoints/035000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..60080824ceb9805d29c885917dc77e0277e52eb2 --- /dev/null +++ b/lerobot-act_so101_test/checkpoints/035000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 35000 +} \ No newline at end of file diff --git a/lerobot-act_so101_test/wandb/debug-internal.log b/lerobot-act_so101_test/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6795b716f5e9e998d3b8ed4c131f334cda8adddb --- /dev/null +++ b/lerobot-act_so101_test/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-12-31T13:18:54.069787725+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-31T13:18:54.476490438+05:30","level":"INFO","msg":"stream: created new stream","id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476511345+05:30","level":"INFO","msg":"stream: started","id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476515165+05:30","level":"INFO","msg":"writer: started","stream_id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476566118+05:30","level":"INFO","msg":"handler: started","stream_id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476575461+05:30","level":"INFO","msg":"sender: started","stream_id":"8ce8thqh"} +{"time":"2025-12-31T16:59:00.732388154+05:30","level":"INFO","msg":"stream: closing","id":"8ce8thqh"} +{"time":"2025-12-31T16:59:23.719001417+05:30","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-31T16:59:24.097369588+05:30","level":"INFO","msg":"handler: closed","stream_id":"8ce8thqh"} +{"time":"2025-12-31T16:59:24.097416229+05:30","level":"INFO","msg":"sender: closed","stream_id":"8ce8thqh"} +{"time":"2025-12-31T16:59:24.097422049+05:30","level":"INFO","msg":"stream: closed","id":"8ce8thqh"} diff --git a/lerobot-act_so101_test/wandb/debug.log b/lerobot-act_so101_test/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0b6b9a9c003a83550ca41013864d666b422096df --- /dev/null +++ b/lerobot-act_so101_test/wandb/debug.log @@ -0,0 +1,23 @@ +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Configure stats pid to 1804411 +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug.log +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-internal.log +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:init():813] calling init triggers +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'lerobot-act', 'root': '/home/zeux/lerobot-act-dataset', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'local/lerobot-act-so101-model', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'chunk_size': 100, 'n_action_steps': 100, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/lerobot-act_so101_test', 'job_name': 'act', 'resume': False, 'seed': 1000, 'num_workers': 8, 'batch_size': 64, 'steps': 35000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:init():854] starting backend +2025-12-31 13:18:54,065 INFO MainThread:1804411 [wandb_init.py:init():857] sending inform_init request +2025-12-31 13:18:54,067 INFO MainThread:1804411 [wandb_init.py:init():865] backend started and connected +2025-12-31 13:18:54,068 INFO MainThread:1804411 [wandb_init.py:init():936] updated telemetry +2025-12-31 13:18:54,069 INFO MainThread:1804411 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-31 13:18:54,935 INFO MainThread:1804411 [wandb_init.py:init():1011] starting run threads in backend +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_console_start():2506] atexit reg +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-31 13:18:54,996 INFO MainThread:1804411 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-31 16:59:00,732 INFO wandb-AsyncioManager-main:1804411 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-31 16:59:00,732 INFO wandb-AsyncioManager-main:1804411 [mailbox.py:close():137] Closing mailbox, abandoning 2 handles. diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/config.yaml b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..169e5fc44d5d9dde3b50c4a95e14796e4d5bf09d --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/config.yaml @@ -0,0 +1,227 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + nxltchl4gaduvozis0gz9xveosym3k7a: + args: + - --dataset.repo_id=lerobot-act + - --dataset.root=/home/zeux/lerobot-act-dataset + - --dataset.video_backend=pyav + - --policy.type=act + - --policy.repo_id=local/lerobot-act-so101-model + - --output_dir=outputs/train/lerobot-act_so101_test + - --policy.device=cuda + - --policy.use_amp=true + - --batch_size=64 + - --num_workers=8 + - --steps=35000 + - --wandb.enable=true + - --policy.push_to_hub=true + codePath: miniconda3/envs/lerobot/bin/lerobot-train + codePathLocal: miniconda3/envs/lerobot/bin/lerobot-train + cpu_count: 24 + cpu_count_logical: 24 + cudaVersion: "13.0" + disk: + /: + total: "200449576960" + used: "34473017344" + executable: /home/zeux/miniconda3/envs/lerobot/bin/python3.10 + gpu: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + gpu_count: 1 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + uuid: GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e + host: zeux + memory: + total: "134489886720" + os: Linux-6.8.0-90-generic-x86_64-with-glibc2.35 + program: /home/zeux/miniconda3/envs/lerobot/bin/lerobot-train + python: CPython 3.10.19 + root: outputs/train/lerobot-act_so101_test + startedAt: "2025-12-31T07:48:53.840056Z" + writerId: nxltchl4gaduvozis0gz9xveosym3k7a + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "3": + - 13 + - 15 + - 16 + - 61 + "4": 3.10.19 + "5": 0.21.4 + "6": 4.57.3 + "10": + - 21 + "12": 0.21.4 + "13": linux-x86_64 +batch_size: + value: 64 +checkpoint_path: + value: null +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: lerobot-act + revision: null + root: /home/zeux/lerobot-act-dataset + streaming: false + use_imagenet_stats: true + video_backend: pyav +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: act +log_freq: + value: 200 +num_workers: + value: 8 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 1e-05 + type: adamw + weight_decay: 0.0001 +output_dir: + value: outputs/train/lerobot-act_so101_test +policy: + value: + chunk_size: 100 + device: cuda + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + kl_weight: 10 + latent_dim: 32 + license: null + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: MEAN_STD + optimizer_lr: 1e-05 + optimizer_lr_backbone: 1e-05 + optimizer_weight_decay: 0.0001 + pre_norm: false + pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1 + pretrained_path: null + private: null + push_to_hub: true + replace_final_stride_with_dilation: false + repo_id: local/lerobot-act-so101-model + tags: null + temporal_ensemble_coeff: null + type: act + use_amp: true + use_vae: true + vision_backbone: resnet18 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 20000 +scheduler: + value: null +seed: + value: 1000 +steps: + value: 35000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/output.log b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..879d410d1fbbb82e07bc0aeb30b87b41c7f76132 --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/output.log @@ -0,0 +1,257 @@ +INFO 2025-12-31 13:18:54 db_utils.py:102 Logs will be synced with wandb. +INFO 2025-12-31 13:18:54 db_utils.py:103 Track this run --> https://wandb.ai/nwaves-ai-nwaves/lerobot/runs/8ce8thqh +INFO 2025-12-31 13:18:54 ot_train.py:183 Creating dataset +INFO 2025-12-31 13:18:55 ot_train.py:202 Creating policy +INFO 2025-12-31 13:18:55 ot_train.py:247 Creating optimizer and scheduler +INFO 2025-12-31 13:18:55 ot_train.py:259 Output dir: outputs/train/lerobot-act_so101_test +INFO 2025-12-31 13:18:55 ot_train.py:264 cfg.steps=35000 (35K) +INFO 2025-12-31 13:18:55 ot_train.py:265 dataset.num_frames=35618 (36K) +INFO 2025-12-31 13:18:55 ot_train.py:266 dataset.num_episodes=40 +INFO 2025-12-31 13:18:55 ot_train.py:269 Effective batch size: 64 x 1 = 64 +INFO 2025-12-31 13:18:55 ot_train.py:270 num_learnable_params=51597190 (52M) +INFO 2025-12-31 13:18:55 ot_train.py:271 num_total_params=51597190 (52M) +INFO 2025-12-31 13:18:55 ot_train.py:327 Start offline training on a fixed dataset +INFO 2025-12-31 13:20:16 ot_train.py:354 step:200 smpl:13K ep:14 epch:0.36 loss:5.354 grdn:79.461 lr:1.0e-05 updt_s:0.366 data_s:0.037 +INFO 2025-12-31 13:21:29 ot_train.py:354 step:400 smpl:26K ep:29 epch:0.72 loss:2.042 grdn:35.399 lr:1.0e-05 updt_s:0.351 data_s:0.015 +INFO 2025-12-31 13:22:48 ot_train.py:354 step:600 smpl:38K ep:43 epch:1.08 loss:1.576 grdn:32.378 lr:1.0e-05 updt_s:0.362 data_s:0.031 +INFO 2025-12-31 13:24:02 ot_train.py:354 step:800 smpl:51K ep:57 epch:1.44 loss:1.250 grdn:28.865 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 13:25:16 ot_train.py:354 step:1K smpl:64K ep:72 epch:1.80 loss:1.008 grdn:26.938 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:26:33 ot_train.py:354 step:1K smpl:77K ep:86 epch:2.16 loss:0.811 grdn:24.199 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 13:27:48 ot_train.py:354 step:1K smpl:90K ep:101 epch:2.52 loss:0.661 grdn:22.866 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:29:02 ot_train.py:354 step:2K smpl:102K ep:115 epch:2.87 loss:0.540 grdn:20.452 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:30:19 ot_train.py:354 step:2K smpl:115K ep:129 epch:3.23 loss:0.450 grdn:18.978 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 13:31:34 ot_train.py:354 step:2K smpl:128K ep:144 epch:3.59 loss:0.380 grdn:17.760 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:32:48 ot_train.py:354 step:2K smpl:141K ep:158 epch:3.95 loss:0.331 grdn:17.224 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:34:05 ot_train.py:354 step:2K smpl:154K ep:172 epch:4.31 loss:0.297 grdn:16.458 lr:1.0e-05 updt_s:0.354 data_s:0.028 +INFO 2025-12-31 13:35:19 ot_train.py:354 step:3K smpl:166K ep:187 epch:4.67 loss:0.268 grdn:16.033 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:36:36 ot_train.py:354 step:3K smpl:179K ep:201 epch:5.03 loss:0.248 grdn:15.266 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 13:37:51 ot_train.py:354 step:3K smpl:192K ep:216 epch:5.39 loss:0.232 grdn:15.217 lr:1.0e-05 updt_s:0.356 data_s:0.017 +INFO 2025-12-31 13:39:06 ot_train.py:354 step:3K smpl:205K ep:230 epch:5.75 loss:0.216 grdn:14.124 lr:1.0e-05 updt_s:0.356 data_s:0.016 +INFO 2025-12-31 13:40:23 ot_train.py:354 step:3K smpl:218K ep:244 epch:6.11 loss:0.207 grdn:13.907 lr:1.0e-05 updt_s:0.354 data_s:0.034 +INFO 2025-12-31 13:41:38 ot_train.py:354 step:4K smpl:230K ep:259 epch:6.47 loss:0.191 grdn:13.139 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 13:42:52 ot_train.py:354 step:4K smpl:243K ep:273 epch:6.83 loss:0.187 grdn:13.581 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 13:44:10 ot_train.py:354 step:4K smpl:256K ep:287 epch:7.19 loss:0.178 grdn:12.612 lr:1.0e-05 updt_s:0.354 data_s:0.034 +INFO 2025-12-31 13:45:25 ot_train.py:354 step:4K smpl:269K ep:302 epch:7.55 loss:0.170 grdn:12.367 lr:1.0e-05 updt_s:0.357 data_s:0.017 +INFO 2025-12-31 13:46:39 ot_train.py:354 step:4K smpl:282K ep:316 epch:7.91 loss:0.163 grdn:12.463 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 13:47:57 ot_train.py:354 step:5K smpl:294K ep:331 epch:8.27 loss:0.158 grdn:12.824 lr:1.0e-05 updt_s:0.355 data_s:0.033 +INFO 2025-12-31 13:49:11 ot_train.py:354 step:5K smpl:307K ep:345 epch:8.62 loss:0.154 grdn:11.798 lr:1.0e-05 updt_s:0.357 data_s:0.016 +INFO 2025-12-31 13:50:25 ot_train.py:354 step:5K smpl:320K ep:359 epch:8.98 loss:0.148 grdn:12.203 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 13:51:43 ot_train.py:354 step:5K smpl:333K ep:374 epch:9.34 loss:0.143 grdn:11.829 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 13:52:57 ot_train.py:354 step:5K smpl:346K ep:388 epch:9.70 loss:0.141 grdn:12.040 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:54:14 ot_train.py:354 step:6K smpl:358K ep:402 epch:10.06 loss:0.133 grdn:11.093 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 13:55:29 ot_train.py:354 step:6K smpl:371K ep:417 epch:10.42 loss:0.130 grdn:10.938 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:56:43 ot_train.py:354 step:6K smpl:384K ep:431 epch:10.78 loss:0.130 grdn:11.160 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 13:58:00 ot_train.py:354 step:6K smpl:397K ep:446 epch:11.14 loss:0.126 grdn:10.801 lr:1.0e-05 updt_s:0.354 data_s:0.030 +INFO 2025-12-31 13:59:14 ot_train.py:354 step:6K smpl:410K ep:460 epch:11.50 loss:0.121 grdn:10.905 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:00:29 ot_train.py:354 step:7K smpl:422K ep:474 epch:11.86 loss:0.120 grdn:10.922 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:01:46 ot_train.py:354 step:7K smpl:435K ep:489 epch:12.22 loss:0.116 grdn:9.954 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 14:03:00 ot_train.py:354 step:7K smpl:448K ep:503 epch:12.58 loss:0.113 grdn:9.952 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:04:15 ot_train.py:354 step:7K smpl:461K ep:517 epch:12.94 loss:0.111 grdn:10.116 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:05:32 ot_train.py:354 step:7K smpl:474K ep:532 epch:13.30 loss:0.110 grdn:10.374 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 14:06:46 ot_train.py:354 step:8K smpl:486K ep:546 epch:13.66 loss:0.107 grdn:10.180 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:08:03 ot_train.py:354 step:8K smpl:499K ep:561 epch:14.02 loss:0.106 grdn:10.077 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 14:09:18 ot_train.py:354 step:8K smpl:512K ep:575 epch:14.37 loss:0.102 grdn:9.894 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:10:32 ot_train.py:354 step:8K smpl:525K ep:589 epch:14.73 loss:0.101 grdn:10.146 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:11:49 ot_train.py:354 step:8K smpl:538K ep:604 epch:15.09 loss:0.100 grdn:9.853 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 14:13:04 ot_train.py:354 step:9K smpl:550K ep:618 epch:15.45 loss:0.097 grdn:9.097 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 14:14:18 ot_train.py:354 step:9K smpl:563K ep:632 epch:15.81 loss:0.098 grdn:9.567 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:15:35 ot_train.py:354 step:9K smpl:576K ep:647 epch:16.17 loss:0.094 grdn:9.654 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 14:16:50 ot_train.py:354 step:9K smpl:589K ep:661 epch:16.53 loss:0.095 grdn:9.516 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 14:18:04 ot_train.py:354 step:9K smpl:602K ep:676 epch:16.89 loss:0.092 grdn:9.076 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 14:19:21 ot_train.py:354 step:10K smpl:614K ep:690 epch:17.25 loss:0.091 grdn:8.884 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 14:20:36 ot_train.py:354 step:10K smpl:627K ep:704 epch:17.61 loss:0.090 grdn:8.930 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:21:50 ot_train.py:354 step:10K smpl:640K ep:719 epch:17.97 loss:0.086 grdn:8.885 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:23:07 ot_train.py:354 step:10K smpl:653K ep:733 epch:18.33 loss:0.088 grdn:9.240 lr:1.0e-05 updt_s:0.355 data_s:0.033 +INFO 2025-12-31 14:24:22 ot_train.py:354 step:10K smpl:666K ep:747 epch:18.69 loss:0.085 grdn:8.758 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:25:39 ot_train.py:354 step:11K smpl:678K ep:762 epch:19.05 loss:0.085 grdn:8.849 lr:1.0e-05 updt_s:0.355 data_s:0.030 +INFO 2025-12-31 14:26:53 ot_train.py:354 step:11K smpl:691K ep:776 epch:19.41 loss:0.083 grdn:8.831 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:28:07 ot_train.py:354 step:11K smpl:704K ep:791 epch:19.77 loss:0.082 grdn:8.277 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:29:25 ot_train.py:354 step:11K smpl:717K ep:805 epch:20.12 loss:0.082 grdn:8.938 lr:1.0e-05 updt_s:0.355 data_s:0.031 +INFO 2025-12-31 14:30:39 ot_train.py:354 step:11K smpl:730K ep:819 epch:20.48 loss:0.081 grdn:8.285 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:31:53 ot_train.py:354 step:12K smpl:742K ep:834 epch:20.84 loss:0.079 grdn:8.153 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:33:10 ot_train.py:354 step:12K smpl:755K ep:848 epch:21.20 loss:0.079 grdn:8.469 lr:1.0e-05 updt_s:0.355 data_s:0.030 +INFO 2025-12-31 14:34:25 ot_train.py:354 step:12K smpl:768K ep:862 epch:21.56 loss:0.077 grdn:9.393 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:35:39 ot_train.py:354 step:12K smpl:781K ep:877 epch:21.92 loss:0.077 grdn:8.871 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:36:56 ot_train.py:354 step:12K smpl:794K ep:891 epch:22.28 loss:0.076 grdn:7.807 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 14:38:10 ot_train.py:354 step:13K smpl:806K ep:906 epch:22.64 loss:0.075 grdn:7.986 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 14:39:24 ot_train.py:354 step:13K smpl:819K ep:920 epch:23.00 loss:0.076 grdn:8.523 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:40:41 ot_train.py:354 step:13K smpl:832K ep:934 epch:23.36 loss:0.074 grdn:7.770 lr:1.0e-05 updt_s:0.355 data_s:0.029 +INFO 2025-12-31 14:41:56 ot_train.py:354 step:13K smpl:845K ep:949 epch:23.72 loss:0.075 grdn:8.904 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 14:43:13 ot_train.py:354 step:13K smpl:858K ep:963 epch:24.08 loss:0.072 grdn:8.099 lr:1.0e-05 updt_s:0.355 data_s:0.030 +INFO 2025-12-31 14:44:27 ot_train.py:354 step:14K smpl:870K ep:977 epch:24.44 loss:0.072 grdn:7.648 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:45:41 ot_train.py:354 step:14K smpl:883K ep:992 epch:24.80 loss:0.073 grdn:8.119 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 14:46:59 ot_train.py:354 step:14K smpl:896K ep:1K epch:25.16 loss:0.072 grdn:7.931 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 14:48:13 ot_train.py:354 step:14K smpl:909K ep:1K epch:25.52 loss:0.069 grdn:7.301 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:49:27 ot_train.py:354 step:14K smpl:922K ep:1K epch:25.87 loss:0.069 grdn:7.827 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:50:44 ot_train.py:354 step:15K smpl:934K ep:1K epch:26.23 loss:0.070 grdn:8.804 lr:1.0e-05 updt_s:0.354 data_s:0.030 +INFO 2025-12-31 14:51:58 ot_train.py:354 step:15K smpl:947K ep:1K epch:26.59 loss:0.068 grdn:7.208 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:53:13 ot_train.py:354 step:15K smpl:960K ep:1K epch:26.95 loss:0.066 grdn:6.920 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:54:30 ot_train.py:354 step:15K smpl:973K ep:1K epch:27.31 loss:0.067 grdn:7.735 lr:1.0e-05 updt_s:0.355 data_s:0.031 +INFO 2025-12-31 14:55:44 ot_train.py:354 step:15K smpl:986K ep:1K epch:27.67 loss:0.066 grdn:7.494 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:57:01 ot_train.py:354 step:16K smpl:998K ep:1K epch:28.03 loss:0.067 grdn:7.772 lr:1.0e-05 updt_s:0.354 data_s:0.030 +INFO 2025-12-31 14:58:16 ot_train.py:354 step:16K smpl:1M ep:1K epch:28.39 loss:0.066 grdn:7.257 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 14:59:30 ot_train.py:354 step:16K smpl:1M ep:1K epch:28.75 loss:0.065 grdn:7.126 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:00:47 ot_train.py:354 step:16K smpl:1M ep:1K epch:29.11 loss:0.066 grdn:7.749 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 15:02:01 ot_train.py:354 step:16K smpl:1M ep:1K epch:29.47 loss:0.064 grdn:7.243 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:03:16 ot_train.py:354 step:17K smpl:1M ep:1K epch:29.83 loss:0.064 grdn:7.316 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:04:33 ot_train.py:354 step:17K smpl:1M ep:1K epch:30.19 loss:0.062 grdn:6.284 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 15:05:47 ot_train.py:354 step:17K smpl:1M ep:1K epch:30.55 loss:0.062 grdn:7.610 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:07:02 ot_train.py:354 step:17K smpl:1M ep:1K epch:30.91 loss:0.064 grdn:7.469 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:08:19 ot_train.py:354 step:17K smpl:1M ep:1K epch:31.27 loss:0.061 grdn:7.114 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 15:09:33 ot_train.py:354 step:18K smpl:1M ep:1K epch:31.62 loss:0.062 grdn:6.946 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:10:48 ot_train.py:354 step:18K smpl:1M ep:1K epch:31.98 loss:0.063 grdn:8.028 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:12:04 ot_train.py:354 step:18K smpl:1M ep:1K epch:32.34 loss:0.062 grdn:7.480 lr:1.0e-05 updt_s:0.354 data_s:0.030 +INFO 2025-12-31 15:13:19 ot_train.py:354 step:18K smpl:1M ep:1K epch:32.70 loss:0.061 grdn:7.474 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:14:36 ot_train.py:354 step:18K smpl:1M ep:1K epch:33.06 loss:0.060 grdn:6.592 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 15:15:50 ot_train.py:354 step:19K smpl:1M ep:1K epch:33.42 loss:0.059 grdn:6.739 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:17:05 ot_train.py:354 step:19K smpl:1M ep:1K epch:33.78 loss:0.061 grdn:7.565 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 15:18:22 ot_train.py:354 step:19K smpl:1M ep:1K epch:34.14 loss:0.058 grdn:6.927 lr:1.0e-05 updt_s:0.355 data_s:0.031 +INFO 2025-12-31 15:19:36 ot_train.py:354 step:19K smpl:1M ep:1K epch:34.50 loss:0.060 grdn:7.307 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:20:50 ot_train.py:354 step:19K smpl:1M ep:1K epch:34.86 loss:0.058 grdn:6.553 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 15:22:08 ot_train.py:354 step:20K smpl:1M ep:1K epch:35.22 loss:0.058 grdn:6.391 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 15:23:22 ot_train.py:354 step:20K smpl:1M ep:1K epch:35.58 loss:0.057 grdn:6.716 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 15:24:36 ot_train.py:354 step:20K smpl:1M ep:1K epch:35.94 loss:0.058 grdn:7.495 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 15:24:36 ot_train.py:364 Checkpoint policy after step 20000 +INFO 2025-12-31 15:25:55 ot_train.py:354 step:20K smpl:1M ep:1K epch:36.30 loss:0.058 grdn:7.561 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 15:27:10 ot_train.py:354 step:20K smpl:1M ep:1K epch:36.66 loss:0.058 grdn:7.133 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:28:24 ot_train.py:354 step:21K smpl:1M ep:1K epch:37.01 loss:0.057 grdn:6.540 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:29:41 ot_train.py:354 step:21K smpl:1M ep:1K epch:37.37 loss:0.056 grdn:6.810 lr:1.0e-05 updt_s:0.355 data_s:0.031 +INFO 2025-12-31 15:30:55 ot_train.py:354 step:21K smpl:1M ep:2K epch:37.73 loss:0.055 grdn:5.939 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:32:13 ot_train.py:354 step:21K smpl:1M ep:2K epch:38.09 loss:0.057 grdn:7.004 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 15:33:27 ot_train.py:354 step:21K smpl:1M ep:2K epch:38.45 loss:0.054 grdn:5.749 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:34:41 ot_train.py:354 step:22K smpl:1M ep:2K epch:38.81 loss:0.055 grdn:6.297 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:35:58 ot_train.py:354 step:22K smpl:1M ep:2K epch:39.17 loss:0.056 grdn:6.978 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 15:37:13 ot_train.py:354 step:22K smpl:1M ep:2K epch:39.53 loss:0.054 grdn:6.632 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:38:27 ot_train.py:354 step:22K smpl:1M ep:2K epch:39.89 loss:0.055 grdn:6.527 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:39:44 ot_train.py:354 step:22K smpl:1M ep:2K epch:40.25 loss:0.053 grdn:6.137 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 15:40:58 ot_train.py:354 step:23K smpl:1M ep:2K epch:40.61 loss:0.054 grdn:6.441 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 15:42:13 ot_train.py:354 step:23K smpl:1M ep:2K epch:40.97 loss:0.054 grdn:6.459 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 15:43:30 ot_train.py:354 step:23K smpl:1M ep:2K epch:41.33 loss:0.053 grdn:6.023 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 15:44:44 ot_train.py:354 step:23K smpl:1M ep:2K epch:41.69 loss:0.053 grdn:6.146 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:46:02 ot_train.py:354 step:23K smpl:1M ep:2K epch:42.05 loss:0.053 grdn:6.212 lr:1.0e-05 updt_s:0.355 data_s:0.031 +INFO 2025-12-31 15:47:16 ot_train.py:354 step:24K smpl:2M ep:2K epch:42.41 loss:0.052 grdn:6.566 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 15:48:30 ot_train.py:354 step:24K smpl:2M ep:2K epch:42.76 loss:0.052 grdn:6.177 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:49:47 ot_train.py:354 step:24K smpl:2M ep:2K epch:43.12 loss:0.052 grdn:6.439 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 15:51:01 ot_train.py:354 step:24K smpl:2M ep:2K epch:43.48 loss:0.052 grdn:6.557 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:52:16 ot_train.py:354 step:24K smpl:2M ep:2K epch:43.84 loss:0.051 grdn:5.968 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 15:53:33 ot_train.py:354 step:25K smpl:2M ep:2K epch:44.20 loss:0.052 grdn:6.389 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 15:54:47 ot_train.py:354 step:25K smpl:2M ep:2K epch:44.56 loss:0.050 grdn:5.622 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:56:02 ot_train.py:354 step:25K smpl:2M ep:2K epch:44.92 loss:0.051 grdn:6.066 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 15:57:19 ot_train.py:354 step:25K smpl:2M ep:2K epch:45.28 loss:0.051 grdn:6.150 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 15:58:33 ot_train.py:354 step:25K smpl:2M ep:2K epch:45.64 loss:0.050 grdn:6.133 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 15:59:48 ot_train.py:354 step:26K smpl:2M ep:2K epch:46.00 loss:0.050 grdn:5.844 lr:1.0e-05 updt_s:0.355 data_s:0.016 +INFO 2025-12-31 16:01:06 ot_train.py:354 step:26K smpl:2M ep:2K epch:46.36 loss:0.050 grdn:6.100 lr:1.0e-05 updt_s:0.355 data_s:0.034 +INFO 2025-12-31 16:02:20 ot_train.py:354 step:26K smpl:2M ep:2K epch:46.72 loss:0.051 grdn:6.424 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:03:38 ot_train.py:354 step:26K smpl:2M ep:2K epch:47.08 loss:0.049 grdn:5.945 lr:1.0e-05 updt_s:0.355 data_s:0.035 +INFO 2025-12-31 16:04:52 ot_train.py:354 step:26K smpl:2M ep:2K epch:47.44 loss:0.049 grdn:6.103 lr:1.0e-05 updt_s:0.356 data_s:0.016 +INFO 2025-12-31 16:06:07 ot_train.py:354 step:27K smpl:2M ep:2K epch:47.80 loss:0.049 grdn:5.937 lr:1.0e-05 updt_s:0.356 data_s:0.017 +INFO 2025-12-31 16:07:24 ot_train.py:354 step:27K smpl:2M ep:2K epch:48.16 loss:0.049 grdn:5.822 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 16:08:38 ot_train.py:354 step:27K smpl:2M ep:2K epch:48.51 loss:0.049 grdn:6.268 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 16:09:53 ot_train.py:354 step:27K smpl:2M ep:2K epch:48.87 loss:0.048 grdn:5.491 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:11:10 ot_train.py:354 step:27K smpl:2M ep:2K epch:49.23 loss:0.049 grdn:5.944 lr:1.0e-05 updt_s:0.354 data_s:0.032 +INFO 2025-12-31 16:12:25 ot_train.py:354 step:28K smpl:2M ep:2K epch:49.59 loss:0.048 grdn:5.436 lr:1.0e-05 updt_s:0.356 data_s:0.016 +INFO 2025-12-31 16:13:39 ot_train.py:354 step:28K smpl:2M ep:2K epch:49.95 loss:0.047 grdn:5.470 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:14:56 ot_train.py:354 step:28K smpl:2M ep:2K epch:50.31 loss:0.048 grdn:6.592 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 16:16:11 ot_train.py:354 step:28K smpl:2M ep:2K epch:50.67 loss:0.047 grdn:5.629 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:17:25 ot_train.py:354 step:28K smpl:2M ep:2K epch:51.03 loss:0.047 grdn:5.651 lr:1.0e-05 updt_s:0.355 data_s:0.015 +INFO 2025-12-31 16:18:42 ot_train.py:354 step:29K smpl:2M ep:2K epch:51.39 loss:0.048 grdn:5.836 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 16:19:57 ot_train.py:354 step:29K smpl:2M ep:2K epch:51.75 loss:0.047 grdn:5.715 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 16:21:14 ot_train.py:354 step:29K smpl:2M ep:2K epch:52.11 loss:0.048 grdn:5.874 lr:1.0e-05 updt_s:0.354 data_s:0.033 +INFO 2025-12-31 16:22:28 ot_train.py:354 step:29K smpl:2M ep:2K epch:52.47 loss:0.046 grdn:5.131 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:23:43 ot_train.py:354 step:29K smpl:2M ep:2K epch:52.83 loss:0.046 grdn:5.731 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:25:00 ot_train.py:354 step:30K smpl:2M ep:2K epch:53.19 loss:0.047 grdn:5.571 lr:1.0e-05 updt_s:0.354 data_s:0.031 +INFO 2025-12-31 16:26:14 ot_train.py:354 step:30K smpl:2M ep:2K epch:53.55 loss:0.047 grdn:5.913 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 16:27:29 ot_train.py:354 step:30K smpl:2M ep:2K epch:53.91 loss:0.047 grdn:5.821 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:28:46 ot_train.py:354 step:30K smpl:2M ep:2K epch:54.26 loss:0.046 grdn:5.620 lr:1.0e-05 updt_s:0.354 data_s:0.033 +INFO 2025-12-31 16:30:01 ot_train.py:354 step:30K smpl:2M ep:2K epch:54.62 loss:0.045 grdn:5.175 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:31:15 ot_train.py:354 step:31K smpl:2M ep:2K epch:54.98 loss:0.044 grdn:4.907 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:32:32 ot_train.py:354 step:31K smpl:2M ep:2K epch:55.34 loss:0.045 grdn:5.579 lr:1.0e-05 updt_s:0.355 data_s:0.031 +INFO 2025-12-31 16:33:47 ot_train.py:354 step:31K smpl:2M ep:2K epch:55.70 loss:0.045 grdn:5.493 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:35:04 ot_train.py:354 step:31K smpl:2M ep:2K epch:56.06 loss:0.045 grdn:5.320 lr:1.0e-05 updt_s:0.355 data_s:0.031 +INFO 2025-12-31 16:36:18 ot_train.py:354 step:31K smpl:2M ep:2K epch:56.42 loss:0.044 grdn:5.413 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:37:33 ot_train.py:354 step:32K smpl:2M ep:2K epch:56.78 loss:0.044 grdn:5.496 lr:1.0e-05 updt_s:0.357 data_s:0.015 +INFO 2025-12-31 16:38:50 ot_train.py:354 step:32K smpl:2M ep:2K epch:57.14 loss:0.045 grdn:5.232 lr:1.0e-05 updt_s:0.354 data_s:0.033 +INFO 2025-12-31 16:40:04 ot_train.py:354 step:32K smpl:2M ep:2K epch:57.50 loss:0.044 grdn:5.274 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:41:19 ot_train.py:354 step:32K smpl:2M ep:2K epch:57.86 loss:0.045 grdn:5.434 lr:1.0e-05 updt_s:0.356 data_s:0.016 +INFO 2025-12-31 16:42:36 ot_train.py:354 step:32K smpl:2M ep:2K epch:58.22 loss:0.044 grdn:5.135 lr:1.0e-05 updt_s:0.355 data_s:0.030 +INFO 2025-12-31 16:43:50 ot_train.py:354 step:33K smpl:2M ep:2K epch:58.58 loss:0.044 grdn:5.474 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:45:05 ot_train.py:354 step:33K smpl:2M ep:2K epch:58.94 loss:0.044 grdn:5.560 lr:1.0e-05 updt_s:0.356 data_s:0.016 +INFO 2025-12-31 16:46:23 ot_train.py:354 step:33K smpl:2M ep:2K epch:59.30 loss:0.044 grdn:5.377 lr:1.0e-05 updt_s:0.355 data_s:0.035 +INFO 2025-12-31 16:47:37 ot_train.py:354 step:33K smpl:2M ep:2K epch:59.66 loss:0.044 grdn:5.795 lr:1.0e-05 updt_s:0.357 data_s:0.016 +INFO 2025-12-31 16:48:52 ot_train.py:354 step:33K smpl:2M ep:2K epch:60.01 loss:0.044 grdn:5.346 lr:1.0e-05 updt_s:0.355 data_s:0.016 +INFO 2025-12-31 16:50:09 ot_train.py:354 step:34K smpl:2M ep:2K epch:60.37 loss:0.043 grdn:5.306 lr:1.0e-05 updt_s:0.355 data_s:0.034 +INFO 2025-12-31 16:51:24 ot_train.py:354 step:34K smpl:2M ep:2K epch:60.73 loss:0.045 grdn:5.791 lr:1.0e-05 updt_s:0.356 data_s:0.016 +INFO 2025-12-31 16:52:42 ot_train.py:354 step:34K smpl:2M ep:2K epch:61.09 loss:0.042 grdn:5.182 lr:1.0e-05 updt_s:0.355 data_s:0.032 +INFO 2025-12-31 16:53:56 ot_train.py:354 step:34K smpl:2M ep:2K epch:61.45 loss:0.042 grdn:5.181 lr:1.0e-05 updt_s:0.356 data_s:0.015 +INFO 2025-12-31 16:55:10 ot_train.py:354 step:34K smpl:2M ep:2K epch:61.81 loss:0.042 grdn:5.000 lr:1.0e-05 updt_s:0.356 data_s:0.016 +INFO 2025-12-31 16:56:29 ot_train.py:354 step:35K smpl:2M ep:2K epch:62.17 loss:0.042 grdn:5.466 lr:1.0e-05 updt_s:0.354 data_s:0.038 +INFO 2025-12-31 16:57:44 ot_train.py:354 step:35K smpl:2M ep:3K epch:62.53 loss:0.042 grdn:5.392 lr:1.0e-05 updt_s:0.357 data_s:0.016 +INFO 2025-12-31 16:58:58 ot_train.py:354 step:35K smpl:2M ep:3K epch:62.89 loss:0.043 grdn:5.261 lr:1.0e-05 updt_s:0.355 data_s:0.016 +INFO 2025-12-31 16:58:58 ot_train.py:364 Checkpoint policy after step 35000 +INFO 2025-12-31 16:59:00 ot_train.py:435 End of training +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 407, in hf_raise_for_status + response.raise_for_status() + File "/home/zeux/.local/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status + raise HTTPError(http_error_msg, response=self) +requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://huggingface.co/api/repos/create + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", line 7, in + sys.exit(main()) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 449, in main + train() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/configs/parser.py", line 233, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 439, in train + unwrapped_policy.push_model_to_hub(cfg) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/policies/pretrained.py", line 211, in push_model_to_hub + repo_id = api.create_repo( + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3779, in create_repo + raise err + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3766, in create_repo + hf_raise_for_status(r) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 471, in hf_raise_for_status + raise _format(HfHubHTTPError, message, response) from e +huggingface_hub.errors.HfHubHTTPError: (Request ID: Root=1-695508fc-3fbeb463197fb585233e2603;49958f1c-88b4-4567-919e-842ecfe126ab) + +403 Forbidden: You don't have the rights to create a model under the namespace "local". +Cannot access content at: https://huggingface.co/api/repos/create. +Make sure your token has the correct permissions. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 407, in hf_raise_for_status + response.raise_for_status() + File "/home/zeux/.local/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status + raise HTTPError(http_error_msg, response=self) +requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://huggingface.co/api/repos/create + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", line 7, in + sys.exit(main()) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 449, in main + train() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/configs/parser.py", line 233, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 439, in train + unwrapped_policy.push_model_to_hub(cfg) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/policies/pretrained.py", line 211, in push_model_to_hub + repo_id = api.create_repo( + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3779, in create_repo + raise err + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3766, in create_repo + hf_raise_for_status(r) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 471, in hf_raise_for_status + raise _format(HfHubHTTPError, message, response) from e +huggingface_hub.errors.HfHubHTTPError: (Request ID: Root=1-695508fc-3fbeb463197fb585233e2603;49958f1c-88b4-4567-919e-842ecfe126ab) + +403 Forbidden: You don't have the rights to create a model under the namespace "local". +Cannot access content at: https://huggingface.co/api/repos/create. +Make sure your token has the correct permissions. diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/requirements.txt b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..69d1fea22d035eea40b25934bb7dc0af20497927 --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/requirements.txt @@ -0,0 +1,257 @@ +nvidia-cusolver==12.0.3.29 +nvidia-curand==10.4.0.35 +anyio==4.12.0 +yarl==1.22.0 +comfyui-workflow-templates-media-other==0.3.15 +tomlkit==0.13.3 +cloudpickle==3.1.2 +typing_extensions==4.15.0 +sentry-sdk==2.48.0 +semver==3.0.4 +mpmath==1.3.0 +draccus==0.10.0 +multidict==6.7.0 +nvidia-cufile-cu12==1.11.1.6 +tokenizers==0.22.1 +GitPython==3.1.45 +datasets==4.1.1 +PySide6_Essentials==6.10.1 +comfyui-workflow-templates-media-image==0.3.17 +Pygments==2.19.2 +aiohappyeyeballs==2.6.1 +protobuf==6.33.2 +PySide6==6.10.1 +comfyui-workflow-templates-core==0.3.12 +PySide2==5.15.2.1 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +scipy==1.15.3 +frozenlist==1.8.0 +nvidia-nvshmem-cu13==3.3.24 +transformers==4.57.3 +typing-inspection==0.4.2 +einops==0.8.1 +shellingham==1.5.4 +Farama-Notifications==0.0.4 +psutil==7.1.3 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +nvidia-cufft==12.0.0.15 +cookiecutter==2.6.0 +rich==14.2.0 +SQLAlchemy==2.0.44 +pydantic_core==2.41.5 +nvidia-nvjitlink==13.0.39 +questionary==2.1.1 +requests==2.32.5 +nvidia-cuda-nvrtc==13.0.48 +torchcodec==0.5 +PyOpenGL-accelerate==3.1.10 +comfyui_frontend_package==1.33.10 +networkx==3.4.2 +nvidia-nvtx==13.0.39 +setuptools==80.9.0 +trampoline==0.1.2 +pandas==2.3.3 +httpx==0.28.1 +av==15.1.0 +comfy-cli==1.5.3 +PyOpenGL==3.1.10 +tomli==2.3.0 +safetensors==0.7.0 +regex==2025.11.3 +wcwidth==0.2.14 +pyyaml-include==1.4.1 +kornia==0.8.2 +packaging==25.0 +nvidia-nccl-cu13==2.27.7 +pydantic==2.12.5 +mixpanel==5.0.0 +markdown-it-py==4.0.0 +multiprocess==0.70.16 +usd-exchange==2.1.0 +h11==0.16.0 +mergedeep==1.3.4 +shiboken6==6.10.1 +text-unidecode==1.3 +deepdiff==8.6.1 +toml==0.10.2 +aiosignal==1.4.0 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +uv==0.9.16 +jsonlines==4.0.0 +xxhash==3.6.0 +numpy==2.2.6 +nvidia-cuda-runtime==13.0.48 +arrow==1.4.0 +spandrel==0.4.1 +nvidia-cuda-cupti==13.0.48 +python-dotenv==1.2.1 +async-timeout==5.0.1 +urllib3==2.6.2 +fsspec==2025.9.0 +nvidia-cusparse==12.6.2.49 +binaryornot==0.4.4 +nvidia-cufile==1.15.0.42 +mdurl==0.1.2 +pydantic-settings==2.12.0 +gitdb==4.0.12 +kornia_rs==0.1.10 +attrs==25.4.0 +mypy_extensions==1.1.0 +tqdm==4.67.1 +alembic==1.17.2 +comfyui-embedded-docs==0.3.1 +python-slugify==8.0.4 +cmake==4.1.3 +smmap==5.0.2 +comfyui-workflow-templates-media-api==0.3.14 +torchsde==0.2.6 +pathspec==0.12.1 +sentencepiece==0.2.1 +charset-normalizer==3.4.4 +nvidia-cusparselt-cu13==0.8.0 +python-xlib==0.33 +ruff==0.14.8 +platformdirs==4.5.1 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-cublas==13.0.0.19 +prompt_toolkit==3.0.52 +ImageIO==2.37.2 +termcolor==3.3.0 +wandb==0.21.4 +annotated-types==0.7.0 +comfyui-workflow-templates-media-video==0.3.12 +typer==0.20.0 +dill==0.4.0 +propcache==0.4.1 +typing-inspect==0.9.0 +greenlet==3.3.0 +hf-xet==1.2.0 +shiboken2==5.15.2.1 +tzdata==2025.2 +aiohttp==3.13.2 +websocket-client==1.9.0 +nvidia-cudnn-cu13==9.13.0.50 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +PySide6_Addons==6.10.1 +asgiref==3.11.0 +httpcore==1.0.9 +exceptiongroup==1.3.1 +filelock==3.20.0 +comfyui_workflow_templates==0.7.25 +opencv-python-headless==4.12.0.88 +nvidia-curand-cu12==10.3.7.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-nvtx-cu12==12.6.77 +cloudpickle==3.1.2 +Flask==3.1.2 +pytz==2025.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-cusparselt-cu12==0.7.1 +sentry-sdk==2.48.0 +importlib_metadata==8.7.1 +draccus==0.10.0 +pycparser==2.23 +nvidia-cufile-cu12==1.11.1.6 +numcodecs==0.13.1 +datasets==4.1.1 +nvidia-cusolver-cu12==11.7.1.2 +protobuf==6.33.2 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +setuptools==80.9.0 +asciitree==0.3.3 +antlr4-python3-runtime==4.9.3 +nvidia-cuda-cupti-cu12==12.6.80 +Farama-Notifications==0.0.4 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +PySocks==1.7.1 +torchvision==0.24.1+cu130 +requests==2.32.5 +torchcodec==0.5 +cffi==2.0.0 +itsdangerous==2.2.0 +idna==3.11 +zipp==3.23.0 +pandas==2.3.3 +av==15.1.0 +nvidia-cusparse-cu12==12.5.4.2 +pymunk==6.11.1 +pyyaml-include==1.4.1 +gymnasium==1.2.3 +nvidia-cuda-runtime-cu12==12.6.77 +h5py==3.15.1 +triton==3.6.0+git9844da95 +triton==3.5.1 +multiprocess==0.70.16 +lerobot==0.4.2 +Jinja2==3.1.6 +click==8.3.1 +nvidia-nccl-cu12==2.28.9 +nvidia-nccl-cu12==2.26.2 +mergedeep==1.3.4 +deepdiff==8.6.1 +toml==0.10.2 +sympy==1.14.0 +hf_transfer==0.1.9 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +soupsieve==2.8.1 +jsonlines==4.0.0 +xxhash==3.6.0 +pyzmq==27.1.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +omegaconf==2.3.0 +Werkzeug==3.1.4 +urllib3==2.6.2 +fsspec==2025.9.0 +pillow==12.0.0 +pytorch-triton==3.1.0+cf34004b8a +cuda-pathfinder==1.2.2 +gdown==5.2.0 +beautifulsoup4==4.14.3 +mypy_extensions==1.1.0 +numba==0.63.1 +cmake==4.1.3 +certifi==2025.11.12 +num2words==0.5.14 +fasteners==0.20 +python-xlib==0.33 +nvidia-nvjitlink-cu12==12.6.85 +platformdirs==4.5.1 +torch==2.9.1+cu130 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-nvshmem-cu12==3.4.5 +ImageIO==2.37.2 +termcolor==3.2.0 +zarr==2.18.3 +six==1.17.0 +blinker==1.9.0 +wandb==0.21.4 +MarkupSafe==3.0.3 +wheel==0.45.1 +dill==0.4.0 +typing-inspect==0.9.0 +cuda-bindings==12.9.4 +pip==25.3 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +nvidia-cublas-cu12==12.6.4.1 +docopt==0.6.2 +llvmlite==0.46.0 +nvidia-cufft-cu12==11.3.0.4 +opencv-python-headless==4.12.0.88 diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/wandb-metadata.json b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fa2dc9363d4a09b96e80d2a57ab9b0084f215ae7 --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/wandb-metadata.json @@ -0,0 +1,50 @@ +{ + "os": "Linux-6.8.0-90-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2025-12-31T07:48:53.840056Z", + "args": [ + "--dataset.repo_id=lerobot-act", + "--dataset.root=/home/zeux/lerobot-act-dataset", + "--dataset.video_backend=pyav", + "--policy.type=act", + "--policy.repo_id=local/lerobot-act-so101-model", + "--output_dir=outputs/train/lerobot-act_so101_test", + "--policy.device=cuda", + "--policy.use_amp=true", + "--batch_size=64", + "--num_workers=8", + "--steps=35000", + "--wandb.enable=true", + "--policy.push_to_hub=true" + ], + "program": "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", + "codePath": "miniconda3/envs/lerobot/bin/lerobot-train", + "codePathLocal": "miniconda3/envs/lerobot/bin/lerobot-train", + "root": "outputs/train/lerobot-act_so101_test", + "host": "zeux", + "executable": "/home/zeux/miniconda3/envs/lerobot/bin/python3.10", + "cpu_count": 24, + "cpu_count_logical": 24, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "gpu_count": 1, + "disk": { + "/": { + "total": "200449576960", + "used": "34473017344" + } + }, + "memory": { + "total": "134489886720" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e" + } + ], + "cudaVersion": "13.0", + "writerId": "nxltchl4gaduvozis0gz9xveosym3k7a" +} \ No newline at end of file diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/wandb-summary.json b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ba89cd92f998e50b6b0a2d61a9e0ce149db917e3 --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/loss":0.04274633360095322,"_wandb":{"runtime":13205},"train/epochs":62.8895502274131,"train/samples":2240000,"_runtime":13205.796850913,"train/kld_loss":9.622704237699509e-05,"train/update_s":0.35532118829112735,"train/grad_norm":5.261109675168991,"train/lr":1.0000000000000021e-05,"train/steps":35000,"_timestamp":1.7671805384628017e+09,"train/episodes":2515.582009096524,"train/l1_loss":0.04637971520423889,"train/dataloading_s":0.016312788063078187,"_step":35000} \ No newline at end of file diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-core.log b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..7f9fe349060801b061c96b6a17d05e90e1843a4c --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2025-12-31T13:18:53.875818977+05:30","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpgumkqn5p/port-1804411.txt","pid":1804411,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-12-31T13:18:53.876237277+05:30","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1804411-1804472-673694980/socket","Net":"unix"}} +{"time":"2025-12-31T13:18:53.876297602+05:30","level":"INFO","msg":"server: will exit if parent process dies","ppid":1804411} +{"time":"2025-12-31T13:18:54.06607548+05:30","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-12-31T13:18:54.069707541+05:30","level":"INFO","msg":"handleInformInit: received","streamId":"8ce8thqh","id":"1(@)"} +{"time":"2025-12-31T13:18:54.476514062+05:30","level":"INFO","msg":"handleInformInit: stream started","streamId":"8ce8thqh","id":"1(@)"} +{"time":"2025-12-31T16:59:00.732355125+05:30","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-12-31T16:59:00.732432869+05:30","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-12-31T16:59:00.732457152+05:30","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-12-31T16:59:00.732459909+05:30","level":"INFO","msg":"server is shutting down"} +{"time":"2025-12-31T16:59:00.732491666+05:30","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1804411-1804472-673694980/socket","Net":"unix"}} +{"time":"2025-12-31T16:59:23.668865223+05:30","level":"ERROR","msg":"processOutgoingData: flush error","error":"write unix /tmp/wandb-1804411-1804472-673694980/socket->@: use of closed network connection","id":"1(@)"} +{"time":"2025-12-31T16:59:24.106575344+05:30","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-12-31T16:59:24.10659737+05:30","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-12-31T16:59:24.10660352+05:30","level":"INFO","msg":"server is closed"} diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-internal.log b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6795b716f5e9e998d3b8ed4c131f334cda8adddb --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-12-31T13:18:54.069787725+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-31T13:18:54.476490438+05:30","level":"INFO","msg":"stream: created new stream","id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476511345+05:30","level":"INFO","msg":"stream: started","id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476515165+05:30","level":"INFO","msg":"writer: started","stream_id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476566118+05:30","level":"INFO","msg":"handler: started","stream_id":"8ce8thqh"} +{"time":"2025-12-31T13:18:54.476575461+05:30","level":"INFO","msg":"sender: started","stream_id":"8ce8thqh"} +{"time":"2025-12-31T16:59:00.732388154+05:30","level":"INFO","msg":"stream: closing","id":"8ce8thqh"} +{"time":"2025-12-31T16:59:23.719001417+05:30","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-31T16:59:24.097369588+05:30","level":"INFO","msg":"handler: closed","stream_id":"8ce8thqh"} +{"time":"2025-12-31T16:59:24.097416229+05:30","level":"INFO","msg":"sender: closed","stream_id":"8ce8thqh"} +{"time":"2025-12-31T16:59:24.097422049+05:30","level":"INFO","msg":"stream: closed","id":"8ce8thqh"} diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug.log b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0b6b9a9c003a83550ca41013864d666b422096df --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug.log @@ -0,0 +1,23 @@ +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Configure stats pid to 1804411 +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug.log +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/logs/debug-internal.log +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:init():813] calling init triggers +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'lerobot-act', 'root': '/home/zeux/lerobot-act-dataset', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'local/lerobot-act-so101-model', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'chunk_size': 100, 'n_action_steps': 100, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/lerobot-act_so101_test', 'job_name': 'act', 'resume': False, 'seed': 1000, 'num_workers': 8, 'batch_size': 64, 'steps': 35000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-31 13:18:53,862 INFO MainThread:1804411 [wandb_init.py:init():854] starting backend +2025-12-31 13:18:54,065 INFO MainThread:1804411 [wandb_init.py:init():857] sending inform_init request +2025-12-31 13:18:54,067 INFO MainThread:1804411 [wandb_init.py:init():865] backend started and connected +2025-12-31 13:18:54,068 INFO MainThread:1804411 [wandb_init.py:init():936] updated telemetry +2025-12-31 13:18:54,069 INFO MainThread:1804411 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-31 13:18:54,935 INFO MainThread:1804411 [wandb_init.py:init():1011] starting run threads in backend +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_console_start():2506] atexit reg +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-31 13:18:54,995 INFO MainThread:1804411 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-31 13:18:54,996 INFO MainThread:1804411 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-31 16:59:00,732 INFO wandb-AsyncioManager-main:1804411 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-31 16:59:00,732 INFO wandb-AsyncioManager-main:1804411 [mailbox.py:close():137] Closing mailbox, abandoning 2 handles. diff --git a/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/run-8ce8thqh.wandb b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/run-8ce8thqh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1a5fdd214d7bf18ed001c5ccc862933eab202f85 --- /dev/null +++ b/lerobot-act_so101_test/wandb/run-20251231_131853-8ce8thqh/run-8ce8thqh.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d903884c62959749ee1e3cceed6567cfad77a4541b94b21994eb8b12fc9e4a +size 958256 diff --git a/lerobot_diffusion_model/checkpoints/020000/pretrained_model/config.json b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4a997f884ae437256b37044a7e2392d1b3259a8a --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/config.json @@ -0,0 +1,82 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "kabilanKB/diffusion-lerobotact", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "horizon": 16, + "n_action_steps": 8, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 7, + "vision_backbone": "resnet18", + "crop_shape": [ + 84, + 84 + ], + "crop_is_random": true, + "pretrained_backbone_weights": null, + "use_group_norm": true, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500 +} \ No newline at end of file diff --git a/lerobot_diffusion_model/checkpoints/020000/pretrained_model/model.safetensors b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ca68ae7ef231e2a957bbd95bccbd12a32b95400d --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88e40741113e5628491fda2973d1a1b6a41c92dc9643e41ce5f07d770474f933 +size 1051836312 diff --git a/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_postprocessor.json b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..f2c5e6cea7850bb999ab193f96021fa0129ce4db --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..85a57f68a0f378553fb7925dd57f1b6690197ba8 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a521cd2d101c29b05f1a61273cce28a42771c079c5568672627c9c96c355c7f8 +size 6544 diff --git a/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_preprocessor.json b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..fd33104f3ab160567a415724a8e144d39ea9be11 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,56 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..85a57f68a0f378553fb7925dd57f1b6690197ba8 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a521cd2d101c29b05f1a61273cce28a42771c079c5568672627c9c96c355c7f8 +size 6544 diff --git a/lerobot_diffusion_model/checkpoints/020000/pretrained_model/train_config.json b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..667928f0bd49908466716e59eb6aa822f46d64c6 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "kabilanKB/lerobotact-dataset", + "root": null, + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav", + "streaming": false + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "kabilanKB/diffusion-lerobotact", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "horizon": 16, + "n_action_steps": 8, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 7, + "vision_backbone": "resnet18", + "crop_shape": [ + 84, + 84 + ], + "crop_is_random": true, + "pretrained_backbone_weights": null, + "use_group_norm": true, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500 + }, + "output_dir": "outputs/train/lerobot_diffusion_model", + "job_name": "diffusion", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 64, + "steps": 20000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "pkxhwzf2", + "mode": null + }, + "checkpoint_path": null, + "rename_map": {} +} \ No newline at end of file diff --git a/lerobot_diffusion_model/checkpoints/020000/training_state/optimizer_param_groups.json b/lerobot_diffusion_model/checkpoints/020000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..5ad4a0652f3e90db3740c824423694349bd048f0 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/training_state/optimizer_param_groups.json @@ -0,0 +1,233 @@ +[ + { + "lr": 0.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": false, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211 + ] + } +] \ No newline at end of file diff --git a/lerobot_diffusion_model/checkpoints/020000/training_state/optimizer_state.safetensors b/lerobot_diffusion_model/checkpoints/020000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0adcf77b7db218ad07c7432013e31400e1dce0ce --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e621dffcdc4f667e475c924d1eb8872ba639e1ef3f5db0ef8258885e0cda377 +size 2103678952 diff --git a/lerobot_diffusion_model/checkpoints/020000/training_state/rng_state.safetensors b/lerobot_diffusion_model/checkpoints/020000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9b117dd4b0eb40766016d0341b10e9851762ac30 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f8308a4ab29e571406935de1c16613a89991fa68bd7e60b79049204005ccbe2 +size 15708 diff --git a/lerobot_diffusion_model/checkpoints/020000/training_state/scheduler_state.json b/lerobot_diffusion_model/checkpoints/020000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..84281984fcabf796762d5aa4110d80140211a135 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 20000, + "_step_count": 20001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/lerobot_diffusion_model/checkpoints/020000/training_state/training_step.json b/lerobot_diffusion_model/checkpoints/020000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9bb47026c5d5237ca6fc5dbff6020dd122ea05 --- /dev/null +++ b/lerobot_diffusion_model/checkpoints/020000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 20000 +} \ No newline at end of file diff --git a/lerobot_diffusion_model/wandb/debug-internal.log b/lerobot_diffusion_model/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..435ff0e7547bc21b79e4668fb4e04f9cc575ef78 --- /dev/null +++ b/lerobot_diffusion_model/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-12-31T10:57:56.017911213+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-31T10:57:56.434086163+05:30","level":"INFO","msg":"stream: created new stream","id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.434103309+05:30","level":"INFO","msg":"stream: started","id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.434140193+05:30","level":"INFO","msg":"writer: started","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.434285297+05:30","level":"INFO","msg":"handler: started","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.43430051+05:30","level":"INFO","msg":"sender: started","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:57.174973936+05:30","level":"INFO","msg":"stream: closing","id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:59.116638777+05:30","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-31T12:39:59.491193968+05:30","level":"INFO","msg":"handler: closed","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:59.491242199+05:30","level":"INFO","msg":"sender: closed","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:59.491246801+05:30","level":"INFO","msg":"stream: closed","id":"pkxhwzf2"} diff --git a/lerobot_diffusion_model/wandb/debug.log b/lerobot_diffusion_model/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1deace37fc1132464dcd31b39e80fe83b458e5c5 --- /dev/null +++ b/lerobot_diffusion_model/wandb/debug.log @@ -0,0 +1,23 @@ +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Configure stats pid to 443105 +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug.log +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-internal.log +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:init():813] calling init triggers +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'kabilanKB/lerobotact-dataset', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'kabilanKB/diffusion-lerobotact', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'horizon': 16, 'n_action_steps': 8, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet18', 'crop_shape': [84, 84], 'crop_is_random': True, 'pretrained_backbone_weights': None, 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [512, 1024, 2048], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDPM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': None, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500}, 'output_dir': 'outputs/train/lerobot_diffusion_model', 'job_name': 'diffusion', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 64, 'steps': 20000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:init():854] starting backend +2025-12-31 10:57:56,013 INFO MainThread:443105 [wandb_init.py:init():857] sending inform_init request +2025-12-31 10:57:56,014 INFO MainThread:443105 [wandb_init.py:init():865] backend started and connected +2025-12-31 10:57:56,015 INFO MainThread:443105 [wandb_init.py:init():936] updated telemetry +2025-12-31 10:57:56,016 INFO MainThread:443105 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-31 10:57:56,946 INFO MainThread:443105 [wandb_init.py:init():1011] starting run threads in backend +2025-12-31 10:57:57,005 INFO MainThread:443105 [wandb_run.py:_console_start():2506] atexit reg +2025-12-31 10:57:57,005 INFO MainThread:443105 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-31 10:57:57,006 INFO MainThread:443105 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-31 10:57:57,006 INFO MainThread:443105 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-31 10:57:57,006 INFO MainThread:443105 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-31 12:39:57,174 INFO wandb-AsyncioManager-main:443105 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-31 12:39:57,174 INFO wandb-AsyncioManager-main:443105 [mailbox.py:close():137] Closing mailbox, abandoning 2 handles. diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/config.yaml b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..450eafeded1bbc5bcd91cf6bad076d27f12d4681 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/config.yaml @@ -0,0 +1,245 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + io3tzplzowclvgx723akjha2ri4r13tq: + args: + - --policy.type=diffusion + - --dataset.repo_id=kabilanKB/lerobotact-dataset + - --dataset.video_backend=pyav + - --policy.repo_id=kabilanKB/diffusion-lerobotact + - --policy.push_to_hub=true + - --batch_size=64 + - --steps=20000 + - --output_dir=outputs/train/lerobot_diffusion_model + - --policy.device=cuda + - --policy.use_amp=true + - --wandb.enable=true + codePath: miniconda3/envs/lerobot/bin/lerobot-train + codePathLocal: miniconda3/envs/lerobot/bin/lerobot-train + cpu_count: 24 + cpu_count_logical: 24 + cudaVersion: "13.0" + disk: + /: + total: "200449576960" + used: "34453942272" + executable: /home/zeux/miniconda3/envs/lerobot/bin/python3.10 + gpu: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + gpu_count: 1 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + uuid: GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e + host: zeux + memory: + total: "134489886720" + os: Linux-6.8.0-90-generic-x86_64-with-glibc2.35 + program: /home/zeux/miniconda3/envs/lerobot/bin/lerobot-train + python: CPython 3.10.19 + root: outputs/train/lerobot_diffusion_model + startedAt: "2025-12-31T05:27:55.808088Z" + writerId: io3tzplzowclvgx723akjha2ri4r13tq + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "3": + - 13 + - 15 + - 16 + - 61 + "4": 3.10.19 + "5": 0.21.4 + "6": 4.57.3 + "10": + - 21 + "12": 0.21.4 + "13": linux-x86_64 +batch_size: + value: 64 +checkpoint_path: + value: null +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: kabilanKB/lerobotact-dataset + revision: null + root: null + streaming: false + use_imagenet_stats: true + video_backend: pyav +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: diffusion +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 0.0001 + type: adam + weight_decay: 1e-06 +output_dir: + value: outputs/train/lerobot_diffusion_model +policy: + value: + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + clip_sample_range: 1 + crop_is_random: true + crop_shape: + - 84 + - 84 + device: cuda + diffusion_step_embed_dim: 128 + do_mask_loss_for_padding: false + down_dims: + - 512 + - 1024 + - 2048 + drop_n_last_frames: 7 + horizon: 16 + kernel_size: 5 + license: null + n_action_steps: 8 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler_type: DDPM + normalization_mapping: + ACTION: MIN_MAX + STATE: MIN_MAX + VISUAL: MEAN_STD + num_inference_steps: null + num_train_timesteps: 100 + optimizer_betas: + - 0.95 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-06 + prediction_type: epsilon + pretrained_backbone_weights: null + pretrained_path: null + private: null + push_to_hub: true + repo_id: kabilanKB/diffusion-lerobotact + scheduler_name: cosine + scheduler_warmup_steps: 500 + spatial_softmax_num_keypoints: 32 + tags: null + type: diffusion + use_amp: true + use_film_scale_modulation: true + use_group_norm: true + use_separate_rgb_encoder_per_camera: false + vision_backbone: resnet18 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 20000 +scheduler: + value: + name: cosine + num_warmup_steps: 500 + type: diffuser +seed: + value: 1000 +steps: + value: 20000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/output.log b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..351f8766029e14443bb6ad6152c4b464142abcf1 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/output.log @@ -0,0 +1,125 @@ +INFO 2025-12-31 10:57:57 db_utils.py:102 Logs will be synced with wandb. +INFO 2025-12-31 10:57:57 db_utils.py:103 Track this run --> https://wandb.ai/nwaves-ai-nwaves/lerobot/runs/pkxhwzf2 +INFO 2025-12-31 10:57:57 ot_train.py:183 Creating dataset +INFO 2025-12-31 10:57:57 ot_train.py:202 Creating policy +INFO 2025-12-31 10:57:57 ot_train.py:247 Creating optimizer and scheduler +INFO 2025-12-31 10:57:57 ot_train.py:259 Output dir: outputs/train/lerobot_diffusion_model +INFO 2025-12-31 10:57:57 ot_train.py:264 cfg.steps=20000 (20K) +INFO 2025-12-31 10:57:57 ot_train.py:265 dataset.num_frames=17774 (18K) +INFO 2025-12-31 10:57:57 ot_train.py:266 dataset.num_episodes=20 +INFO 2025-12-31 10:57:57 ot_train.py:269 Effective batch size: 64 x 1 = 64 +INFO 2025-12-31 10:57:57 ot_train.py:270 num_learnable_params=262952742 (263M) +INFO 2025-12-31 10:57:57 ot_train.py:271 num_total_params=262952742 (263M) +INFO 2025-12-31 10:57:57 ot_train.py:327 Start offline training on a fixed dataset +INFO 2025-12-31 10:59:03 ot_train.py:354 step:200 smpl:13K ep:14 epch:0.72 loss:0.547 grdn:3.881 lr:2.0e-05 updt_s:0.071 data_s:0.257 +INFO 2025-12-31 11:00:05 ot_train.py:354 step:400 smpl:26K ep:29 epch:1.44 loss:0.073 grdn:2.565 lr:6.0e-05 updt_s:0.062 data_s:0.246 +INFO 2025-12-31 11:01:01 ot_train.py:354 step:600 smpl:38K ep:43 epch:2.16 loss:0.046 grdn:1.436 lr:9.5e-05 updt_s:0.057 data_s:0.225 +INFO 2025-12-31 11:02:01 ot_train.py:354 step:800 smpl:51K ep:58 epch:2.88 loss:0.034 grdn:0.996 lr:1.0e-04 updt_s:0.057 data_s:0.239 +INFO 2025-12-31 11:03:02 ot_train.py:354 step:1K smpl:64K ep:72 epch:3.60 loss:0.028 grdn:0.855 lr:1.0e-04 updt_s:0.057 data_s:0.251 +INFO 2025-12-31 11:03:59 ot_train.py:354 step:1K smpl:77K ep:86 epch:4.32 loss:0.026 grdn:0.775 lr:1.0e-04 updt_s:0.057 data_s:0.225 +INFO 2025-12-31 11:04:57 ot_train.py:354 step:1K smpl:90K ep:101 epch:5.04 loss:0.023 grdn:0.693 lr:1.0e-04 updt_s:0.057 data_s:0.232 +INFO 2025-12-31 11:05:54 ot_train.py:354 step:2K smpl:102K ep:115 epch:5.76 loss:0.022 grdn:0.652 lr:9.9e-05 updt_s:0.057 data_s:0.231 +INFO 2025-12-31 11:06:55 ot_train.py:354 step:2K smpl:115K ep:130 epch:6.48 loss:0.021 grdn:0.606 lr:9.9e-05 updt_s:0.057 data_s:0.246 +INFO 2025-12-31 11:07:52 ot_train.py:354 step:2K smpl:128K ep:144 epch:7.20 loss:0.019 grdn:0.522 lr:9.9e-05 updt_s:0.057 data_s:0.229 +INFO 2025-12-31 11:08:50 ot_train.py:354 step:2K smpl:141K ep:158 epch:7.92 loss:0.019 grdn:0.565 lr:9.8e-05 updt_s:0.057 data_s:0.231 +INFO 2025-12-31 11:09:49 ot_train.py:354 step:2K smpl:154K ep:173 epch:8.64 loss:0.019 grdn:0.532 lr:9.8e-05 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 11:10:50 ot_train.py:354 step:3K smpl:166K ep:187 epch:9.36 loss:0.018 grdn:0.480 lr:9.7e-05 updt_s:0.057 data_s:0.246 +INFO 2025-12-31 11:11:49 ot_train.py:354 step:3K smpl:179K ep:202 epch:10.08 loss:0.016 grdn:0.444 lr:9.7e-05 updt_s:0.057 data_s:0.235 +INFO 2025-12-31 11:12:47 ot_train.py:354 step:3K smpl:192K ep:216 epch:10.80 loss:0.016 grdn:0.419 lr:9.6e-05 updt_s:0.057 data_s:0.235 +INFO 2025-12-31 11:13:48 ot_train.py:354 step:3K smpl:205K ep:230 epch:11.52 loss:0.016 grdn:0.445 lr:9.6e-05 updt_s:0.057 data_s:0.246 +INFO 2025-12-31 11:14:48 ot_train.py:354 step:3K smpl:218K ep:245 epch:12.24 loss:0.015 grdn:0.403 lr:9.5e-05 updt_s:0.057 data_s:0.242 +INFO 2025-12-31 11:15:48 ot_train.py:354 step:4K smpl:230K ep:259 epch:12.96 loss:0.015 grdn:0.400 lr:9.4e-05 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 11:16:43 ot_train.py:354 step:4K smpl:243K ep:274 epch:13.68 loss:0.015 grdn:0.402 lr:9.3e-05 updt_s:0.058 data_s:0.221 +INFO 2025-12-31 11:17:41 ot_train.py:354 step:4K smpl:256K ep:288 epch:14.40 loss:0.015 grdn:0.371 lr:9.3e-05 updt_s:0.057 data_s:0.232 +INFO 2025-12-31 11:18:42 ot_train.py:354 step:4K smpl:269K ep:302 epch:15.12 loss:0.015 grdn:0.363 lr:9.2e-05 updt_s:0.057 data_s:0.245 +INFO 2025-12-31 11:19:40 ot_train.py:354 step:4K smpl:282K ep:317 epch:15.84 loss:0.014 grdn:0.364 lr:9.1e-05 updt_s:0.057 data_s:0.233 +INFO 2025-12-31 11:20:40 ot_train.py:354 step:5K smpl:294K ep:331 epch:16.56 loss:0.014 grdn:0.344 lr:9.0e-05 updt_s:0.057 data_s:0.245 +INFO 2025-12-31 11:21:39 ot_train.py:354 step:5K smpl:307K ep:346 epch:17.28 loss:0.014 grdn:0.326 lr:8.9e-05 updt_s:0.057 data_s:0.237 +INFO 2025-12-31 11:22:36 ot_train.py:354 step:5K smpl:320K ep:360 epch:18.00 loss:0.014 grdn:0.336 lr:8.8e-05 updt_s:0.057 data_s:0.228 +INFO 2025-12-31 11:23:34 ot_train.py:354 step:5K smpl:333K ep:374 epch:18.72 loss:0.013 grdn:0.329 lr:8.7e-05 updt_s:0.057 data_s:0.228 +INFO 2025-12-31 11:24:33 ot_train.py:354 step:5K smpl:346K ep:389 epch:19.44 loss:0.013 grdn:0.322 lr:8.6e-05 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 11:25:30 ot_train.py:354 step:6K smpl:358K ep:403 epch:20.16 loss:0.013 grdn:0.334 lr:8.5e-05 updt_s:0.057 data_s:0.228 +INFO 2025-12-31 11:26:29 ot_train.py:354 step:6K smpl:371K ep:418 epch:20.88 loss:0.013 grdn:0.318 lr:8.3e-05 updt_s:0.057 data_s:0.237 +INFO 2025-12-31 11:27:26 ot_train.py:354 step:6K smpl:384K ep:432 epch:21.60 loss:0.013 grdn:0.311 lr:8.2e-05 updt_s:0.057 data_s:0.228 +INFO 2025-12-31 11:28:25 ot_train.py:354 step:6K smpl:397K ep:446 epch:22.32 loss:0.013 grdn:0.297 lr:8.1e-05 updt_s:0.057 data_s:0.235 +INFO 2025-12-31 11:29:24 ot_train.py:354 step:6K smpl:410K ep:461 epch:23.04 loss:0.013 grdn:0.310 lr:8.0e-05 updt_s:0.057 data_s:0.237 +INFO 2025-12-31 11:30:19 ot_train.py:354 step:7K smpl:422K ep:475 epch:23.77 loss:0.013 grdn:0.276 lr:7.8e-05 updt_s:0.057 data_s:0.219 +INFO 2025-12-31 11:31:17 ot_train.py:354 step:7K smpl:435K ep:490 epch:24.49 loss:0.012 grdn:0.280 lr:7.7e-05 updt_s:0.057 data_s:0.233 +INFO 2025-12-31 11:32:16 ot_train.py:354 step:7K smpl:448K ep:504 epch:25.21 loss:0.012 grdn:0.278 lr:7.6e-05 updt_s:0.057 data_s:0.237 +INFO 2025-12-31 11:33:16 ot_train.py:354 step:7K smpl:461K ep:519 epch:25.93 loss:0.012 grdn:0.280 lr:7.4e-05 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 11:34:12 ot_train.py:354 step:7K smpl:474K ep:533 epch:26.65 loss:0.012 grdn:0.277 lr:7.3e-05 updt_s:0.057 data_s:0.225 +INFO 2025-12-31 11:35:15 ot_train.py:354 step:8K smpl:486K ep:547 epch:27.37 loss:0.012 grdn:0.277 lr:7.1e-05 updt_s:0.057 data_s:0.255 +INFO 2025-12-31 11:36:13 ot_train.py:354 step:8K smpl:499K ep:562 epch:28.09 loss:0.012 grdn:0.269 lr:7.0e-05 updt_s:0.057 data_s:0.235 +INFO 2025-12-31 11:37:15 ot_train.py:354 step:8K smpl:512K ep:576 epch:28.81 loss:0.011 grdn:0.281 lr:6.8e-05 updt_s:0.057 data_s:0.252 +INFO 2025-12-31 11:38:18 ot_train.py:354 step:8K smpl:525K ep:591 epch:29.53 loss:0.011 grdn:0.238 lr:6.7e-05 updt_s:0.058 data_s:0.257 +INFO 2025-12-31 11:39:27 ot_train.py:354 step:8K smpl:538K ep:605 epch:30.25 loss:0.011 grdn:0.266 lr:6.5e-05 updt_s:0.058 data_s:0.289 +INFO 2025-12-31 11:40:34 ot_train.py:354 step:9K smpl:550K ep:619 epch:30.97 loss:0.011 grdn:0.257 lr:6.4e-05 updt_s:0.057 data_s:0.274 +INFO 2025-12-31 11:41:31 ot_train.py:354 step:9K smpl:563K ep:634 epch:31.69 loss:0.011 grdn:0.272 lr:6.2e-05 updt_s:0.057 data_s:0.229 +INFO 2025-12-31 11:42:31 ot_train.py:354 step:9K smpl:576K ep:648 epch:32.41 loss:0.010 grdn:0.256 lr:6.1e-05 updt_s:0.058 data_s:0.243 +INFO 2025-12-31 11:43:31 ot_train.py:354 step:9K smpl:589K ep:663 epch:33.13 loss:0.010 grdn:0.247 lr:5.9e-05 updt_s:0.057 data_s:0.242 +INFO 2025-12-31 11:44:31 ot_train.py:354 step:9K smpl:602K ep:677 epch:33.85 loss:0.011 grdn:0.253 lr:5.8e-05 updt_s:0.057 data_s:0.243 +INFO 2025-12-31 11:45:29 ot_train.py:354 step:10K smpl:614K ep:691 epch:34.57 loss:0.010 grdn:0.259 lr:5.6e-05 updt_s:0.057 data_s:0.233 +INFO 2025-12-31 11:46:29 ot_train.py:354 step:10K smpl:627K ep:706 epch:35.29 loss:0.010 grdn:0.257 lr:5.4e-05 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 11:47:27 ot_train.py:354 step:10K smpl:640K ep:720 epch:36.01 loss:0.010 grdn:0.241 lr:5.3e-05 updt_s:0.057 data_s:0.232 +INFO 2025-12-31 11:48:25 ot_train.py:354 step:10K smpl:653K ep:735 epch:36.73 loss:0.009 grdn:0.221 lr:5.1e-05 updt_s:0.057 data_s:0.236 +INFO 2025-12-31 11:49:27 ot_train.py:354 step:10K smpl:666K ep:749 epch:37.45 loss:0.009 grdn:0.219 lr:5.0e-05 updt_s:0.057 data_s:0.251 +INFO 2025-12-31 11:50:29 ot_train.py:354 step:11K smpl:678K ep:763 epch:38.17 loss:0.010 grdn:0.240 lr:4.8e-05 updt_s:0.057 data_s:0.253 +INFO 2025-12-31 11:51:30 ot_train.py:354 step:11K smpl:691K ep:778 epch:38.89 loss:0.009 grdn:0.235 lr:4.6e-05 updt_s:0.057 data_s:0.246 +INFO 2025-12-31 11:52:31 ot_train.py:354 step:11K smpl:704K ep:792 epch:39.61 loss:0.009 grdn:0.228 lr:4.5e-05 updt_s:0.057 data_s:0.248 +INFO 2025-12-31 11:53:32 ot_train.py:354 step:11K smpl:717K ep:807 epch:40.33 loss:0.009 grdn:0.216 lr:4.3e-05 updt_s:0.057 data_s:0.247 +INFO 2025-12-31 11:54:30 ot_train.py:354 step:11K smpl:730K ep:821 epch:41.05 loss:0.008 grdn:0.220 lr:4.2e-05 updt_s:0.057 data_s:0.232 +INFO 2025-12-31 11:55:25 ot_train.py:354 step:12K smpl:742K ep:835 epch:41.77 loss:0.009 grdn:0.251 lr:4.0e-05 updt_s:0.057 data_s:0.219 +INFO 2025-12-31 11:56:22 ot_train.py:354 step:12K smpl:755K ep:850 epch:42.49 loss:0.008 grdn:0.204 lr:3.8e-05 updt_s:0.057 data_s:0.229 +INFO 2025-12-31 11:57:20 ot_train.py:354 step:12K smpl:768K ep:864 epch:43.21 loss:0.008 grdn:0.215 lr:3.7e-05 updt_s:0.057 data_s:0.230 +INFO 2025-12-31 11:58:18 ot_train.py:354 step:12K smpl:781K ep:879 epch:43.93 loss:0.008 grdn:0.215 lr:3.5e-05 updt_s:0.057 data_s:0.231 +INFO 2025-12-31 11:59:15 ot_train.py:354 step:12K smpl:794K ep:893 epch:44.65 loss:0.008 grdn:0.195 lr:3.4e-05 updt_s:0.057 data_s:0.229 +INFO 2025-12-31 12:00:15 ot_train.py:354 step:13K smpl:806K ep:907 epch:45.37 loss:0.008 grdn:0.218 lr:3.2e-05 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 12:01:15 ot_train.py:354 step:13K smpl:819K ep:922 epch:46.09 loss:0.008 grdn:0.201 lr:3.1e-05 updt_s:0.057 data_s:0.244 +INFO 2025-12-31 12:02:15 ot_train.py:354 step:13K smpl:832K ep:936 epch:46.81 loss:0.007 grdn:0.191 lr:2.9e-05 updt_s:0.057 data_s:0.243 +INFO 2025-12-31 12:03:14 ot_train.py:354 step:13K smpl:845K ep:951 epch:47.53 loss:0.008 grdn:0.207 lr:2.8e-05 updt_s:0.057 data_s:0.236 +INFO 2025-12-31 12:04:14 ot_train.py:354 step:13K smpl:858K ep:965 epch:48.25 loss:0.007 grdn:0.204 lr:2.6e-05 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 12:05:13 ot_train.py:354 step:14K smpl:870K ep:979 epch:48.97 loss:0.007 grdn:0.195 lr:2.5e-05 updt_s:0.057 data_s:0.239 +INFO 2025-12-31 12:06:12 ot_train.py:354 step:14K smpl:883K ep:994 epch:49.69 loss:0.007 grdn:0.182 lr:2.4e-05 updt_s:0.057 data_s:0.239 +INFO 2025-12-31 12:07:12 ot_train.py:354 step:14K smpl:896K ep:1K epch:50.41 loss:0.007 grdn:0.195 lr:2.2e-05 updt_s:0.057 data_s:0.242 +INFO 2025-12-31 12:08:16 ot_train.py:354 step:14K smpl:909K ep:1K epch:51.13 loss:0.006 grdn:0.185 lr:2.1e-05 updt_s:0.057 data_s:0.259 +INFO 2025-12-31 12:09:15 ot_train.py:354 step:14K smpl:922K ep:1K epch:51.85 loss:0.006 grdn:0.199 lr:2.0e-05 updt_s:0.057 data_s:0.240 +INFO 2025-12-31 12:10:14 ot_train.py:354 step:15K smpl:934K ep:1K epch:52.57 loss:0.006 grdn:0.181 lr:1.8e-05 updt_s:0.057 data_s:0.237 +INFO 2025-12-31 12:11:13 ot_train.py:354 step:15K smpl:947K ep:1K epch:53.29 loss:0.006 grdn:0.171 lr:1.7e-05 updt_s:0.057 data_s:0.238 +INFO 2025-12-31 12:12:12 ot_train.py:354 step:15K smpl:960K ep:1K epch:54.01 loss:0.006 grdn:0.170 lr:1.6e-05 updt_s:0.057 data_s:0.238 +INFO 2025-12-31 12:13:11 ot_train.py:354 step:15K smpl:973K ep:1K epch:54.73 loss:0.006 grdn:0.167 lr:1.5e-05 updt_s:0.057 data_s:0.237 +INFO 2025-12-31 12:14:08 ot_train.py:354 step:15K smpl:986K ep:1K epch:55.45 loss:0.006 grdn:0.169 lr:1.4e-05 updt_s:0.057 data_s:0.229 +INFO 2025-12-31 12:15:07 ot_train.py:354 step:16K smpl:998K ep:1K epch:56.17 loss:0.006 grdn:0.182 lr:1.3e-05 updt_s:0.057 data_s:0.236 +INFO 2025-12-31 12:16:05 ot_train.py:354 step:16K smpl:1M ep:1K epch:56.89 loss:0.005 grdn:0.165 lr:1.2e-05 updt_s:0.057 data_s:0.234 +INFO 2025-12-31 12:17:02 ot_train.py:354 step:16K smpl:1M ep:1K epch:57.61 loss:0.006 grdn:0.174 lr:1.1e-05 updt_s:0.057 data_s:0.227 +INFO 2025-12-31 12:18:02 ot_train.py:354 step:16K smpl:1M ep:1K epch:58.33 loss:0.005 grdn:0.159 lr:9.5e-06 updt_s:0.057 data_s:0.244 +INFO 2025-12-31 12:18:59 ot_train.py:354 step:16K smpl:1M ep:1K epch:59.05 loss:0.005 grdn:0.160 lr:8.6e-06 updt_s:0.057 data_s:0.229 +INFO 2025-12-31 12:19:58 ot_train.py:354 step:17K smpl:1M ep:1K epch:59.77 loss:0.005 grdn:0.157 lr:7.7e-06 updt_s:0.057 data_s:0.236 +INFO 2025-12-31 12:20:55 ot_train.py:354 step:17K smpl:1M ep:1K epch:60.49 loss:0.005 grdn:0.151 lr:6.9e-06 updt_s:0.057 data_s:0.226 +INFO 2025-12-31 12:21:53 ot_train.py:354 step:17K smpl:1M ep:1K epch:61.21 loss:0.005 grdn:0.147 lr:6.1e-06 updt_s:0.057 data_s:0.235 +INFO 2025-12-31 12:22:53 ot_train.py:354 step:17K smpl:1M ep:1K epch:61.93 loss:0.005 grdn:0.146 lr:5.4e-06 updt_s:0.057 data_s:0.241 +INFO 2025-12-31 12:23:53 ot_train.py:354 step:17K smpl:1M ep:1K epch:62.65 loss:0.005 grdn:0.140 lr:4.7e-06 updt_s:0.057 data_s:0.244 +INFO 2025-12-31 12:24:51 ot_train.py:354 step:18K smpl:1M ep:1K epch:63.37 loss:0.005 grdn:0.140 lr:4.0e-06 updt_s:0.058 data_s:0.229 +INFO 2025-12-31 12:25:50 ot_train.py:354 step:18K smpl:1M ep:1K epch:64.09 loss:0.005 grdn:0.135 lr:3.4e-06 updt_s:0.057 data_s:0.239 +INFO 2025-12-31 12:26:49 ot_train.py:354 step:18K smpl:1M ep:1K epch:64.81 loss:0.005 grdn:0.133 lr:2.8e-06 updt_s:0.057 data_s:0.239 +INFO 2025-12-31 12:27:45 ot_train.py:354 step:18K smpl:1M ep:1K epch:65.53 loss:0.005 grdn:0.131 lr:2.3e-06 updt_s:0.057 data_s:0.222 +INFO 2025-12-31 12:28:43 ot_train.py:354 step:18K smpl:1M ep:1K epch:66.25 loss:0.004 grdn:0.128 lr:1.9e-06 updt_s:0.057 data_s:0.232 +INFO 2025-12-31 12:29:42 ot_train.py:354 step:19K smpl:1M ep:1K epch:66.97 loss:0.004 grdn:0.129 lr:1.5e-06 updt_s:0.057 data_s:0.235 +INFO 2025-12-31 12:30:40 ot_train.py:354 step:19K smpl:1M ep:1K epch:67.69 loss:0.004 grdn:0.124 lr:1.1e-06 updt_s:0.057 data_s:0.236 +INFO 2025-12-31 12:31:39 ot_train.py:354 step:19K smpl:1M ep:1K epch:68.41 loss:0.004 grdn:0.128 lr:7.8e-07 updt_s:0.057 data_s:0.235 +INFO 2025-12-31 12:32:36 ot_train.py:354 step:19K smpl:1M ep:1K epch:69.13 loss:0.004 grdn:0.124 lr:5.3e-07 updt_s:0.057 data_s:0.230 +INFO 2025-12-31 12:33:35 ot_train.py:354 step:19K smpl:1M ep:1K epch:69.85 loss:0.004 grdn:0.124 lr:3.2e-07 updt_s:0.057 data_s:0.237 +INFO 2025-12-31 12:34:35 ot_train.py:354 step:20K smpl:1M ep:1K epch:70.57 loss:0.004 grdn:0.128 lr:1.6e-07 updt_s:0.057 data_s:0.239 +INFO 2025-12-31 12:35:32 ot_train.py:354 step:20K smpl:1M ep:1K epch:71.30 loss:0.004 grdn:0.123 lr:6.0e-08 updt_s:0.057 data_s:0.228 +INFO 2025-12-31 12:36:32 ot_train.py:354 step:20K smpl:1M ep:1K epch:72.02 loss:0.004 grdn:0.124 lr:8.6e-09 updt_s:0.057 data_s:0.243 +INFO 2025-12-31 12:36:32 ot_train.py:364 Checkpoint policy after step 20000 +INFO 2025-12-31 12:36:37 ot_train.py:435 End of training +Processing Files (1 / 1) : 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.05GB / 1.05GB, 5.10MB/s +New Data Upload : 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.05GB / 1.05GB, 5.10MB/s + ...obotact/model.safetensors: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.05GB / 1.05GB +INFO 2025-12-31 12:39:42 etrained.py:237 Model pushed to https://huggingface.co/kabilanKB/diffusion-lerobotact +Processing Files (1 / 1) : 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6.54kB / 6.54kB, 0.00B/s +New Data Upload : | | 0.00B / 0.00B, 0.00B/s + ...zer_processor.safetensors: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6.54kB / 6.54kB +Processing Files (1 / 1) : 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6.54kB / 6.54kB, 0.00B/s +New Data Upload : | | 0.00B / 0.00B, 0.00B/s + ...zer_processor.safetensors: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6.54kB / 6.54kB diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/requirements.txt b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..69d1fea22d035eea40b25934bb7dc0af20497927 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/requirements.txt @@ -0,0 +1,257 @@ +nvidia-cusolver==12.0.3.29 +nvidia-curand==10.4.0.35 +anyio==4.12.0 +yarl==1.22.0 +comfyui-workflow-templates-media-other==0.3.15 +tomlkit==0.13.3 +cloudpickle==3.1.2 +typing_extensions==4.15.0 +sentry-sdk==2.48.0 +semver==3.0.4 +mpmath==1.3.0 +draccus==0.10.0 +multidict==6.7.0 +nvidia-cufile-cu12==1.11.1.6 +tokenizers==0.22.1 +GitPython==3.1.45 +datasets==4.1.1 +PySide6_Essentials==6.10.1 +comfyui-workflow-templates-media-image==0.3.17 +Pygments==2.19.2 +aiohappyeyeballs==2.6.1 +protobuf==6.33.2 +PySide6==6.10.1 +comfyui-workflow-templates-core==0.3.12 +PySide2==5.15.2.1 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +scipy==1.15.3 +frozenlist==1.8.0 +nvidia-nvshmem-cu13==3.3.24 +transformers==4.57.3 +typing-inspection==0.4.2 +einops==0.8.1 +shellingham==1.5.4 +Farama-Notifications==0.0.4 +psutil==7.1.3 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +nvidia-cufft==12.0.0.15 +cookiecutter==2.6.0 +rich==14.2.0 +SQLAlchemy==2.0.44 +pydantic_core==2.41.5 +nvidia-nvjitlink==13.0.39 +questionary==2.1.1 +requests==2.32.5 +nvidia-cuda-nvrtc==13.0.48 +torchcodec==0.5 +PyOpenGL-accelerate==3.1.10 +comfyui_frontend_package==1.33.10 +networkx==3.4.2 +nvidia-nvtx==13.0.39 +setuptools==80.9.0 +trampoline==0.1.2 +pandas==2.3.3 +httpx==0.28.1 +av==15.1.0 +comfy-cli==1.5.3 +PyOpenGL==3.1.10 +tomli==2.3.0 +safetensors==0.7.0 +regex==2025.11.3 +wcwidth==0.2.14 +pyyaml-include==1.4.1 +kornia==0.8.2 +packaging==25.0 +nvidia-nccl-cu13==2.27.7 +pydantic==2.12.5 +mixpanel==5.0.0 +markdown-it-py==4.0.0 +multiprocess==0.70.16 +usd-exchange==2.1.0 +h11==0.16.0 +mergedeep==1.3.4 +shiboken6==6.10.1 +text-unidecode==1.3 +deepdiff==8.6.1 +toml==0.10.2 +aiosignal==1.4.0 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +uv==0.9.16 +jsonlines==4.0.0 +xxhash==3.6.0 +numpy==2.2.6 +nvidia-cuda-runtime==13.0.48 +arrow==1.4.0 +spandrel==0.4.1 +nvidia-cuda-cupti==13.0.48 +python-dotenv==1.2.1 +async-timeout==5.0.1 +urllib3==2.6.2 +fsspec==2025.9.0 +nvidia-cusparse==12.6.2.49 +binaryornot==0.4.4 +nvidia-cufile==1.15.0.42 +mdurl==0.1.2 +pydantic-settings==2.12.0 +gitdb==4.0.12 +kornia_rs==0.1.10 +attrs==25.4.0 +mypy_extensions==1.1.0 +tqdm==4.67.1 +alembic==1.17.2 +comfyui-embedded-docs==0.3.1 +python-slugify==8.0.4 +cmake==4.1.3 +smmap==5.0.2 +comfyui-workflow-templates-media-api==0.3.14 +torchsde==0.2.6 +pathspec==0.12.1 +sentencepiece==0.2.1 +charset-normalizer==3.4.4 +nvidia-cusparselt-cu13==0.8.0 +python-xlib==0.33 +ruff==0.14.8 +platformdirs==4.5.1 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-cublas==13.0.0.19 +prompt_toolkit==3.0.52 +ImageIO==2.37.2 +termcolor==3.3.0 +wandb==0.21.4 +annotated-types==0.7.0 +comfyui-workflow-templates-media-video==0.3.12 +typer==0.20.0 +dill==0.4.0 +propcache==0.4.1 +typing-inspect==0.9.0 +greenlet==3.3.0 +hf-xet==1.2.0 +shiboken2==5.15.2.1 +tzdata==2025.2 +aiohttp==3.13.2 +websocket-client==1.9.0 +nvidia-cudnn-cu13==9.13.0.50 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +PySide6_Addons==6.10.1 +asgiref==3.11.0 +httpcore==1.0.9 +exceptiongroup==1.3.1 +filelock==3.20.0 +comfyui_workflow_templates==0.7.25 +opencv-python-headless==4.12.0.88 +nvidia-curand-cu12==10.3.7.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-nvtx-cu12==12.6.77 +cloudpickle==3.1.2 +Flask==3.1.2 +pytz==2025.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-cusparselt-cu12==0.7.1 +sentry-sdk==2.48.0 +importlib_metadata==8.7.1 +draccus==0.10.0 +pycparser==2.23 +nvidia-cufile-cu12==1.11.1.6 +numcodecs==0.13.1 +datasets==4.1.1 +nvidia-cusolver-cu12==11.7.1.2 +protobuf==6.33.2 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +setuptools==80.9.0 +asciitree==0.3.3 +antlr4-python3-runtime==4.9.3 +nvidia-cuda-cupti-cu12==12.6.80 +Farama-Notifications==0.0.4 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +PySocks==1.7.1 +torchvision==0.24.1+cu130 +requests==2.32.5 +torchcodec==0.5 +cffi==2.0.0 +itsdangerous==2.2.0 +idna==3.11 +zipp==3.23.0 +pandas==2.3.3 +av==15.1.0 +nvidia-cusparse-cu12==12.5.4.2 +pymunk==6.11.1 +pyyaml-include==1.4.1 +gymnasium==1.2.3 +nvidia-cuda-runtime-cu12==12.6.77 +h5py==3.15.1 +triton==3.6.0+git9844da95 +triton==3.5.1 +multiprocess==0.70.16 +lerobot==0.4.2 +Jinja2==3.1.6 +click==8.3.1 +nvidia-nccl-cu12==2.28.9 +nvidia-nccl-cu12==2.26.2 +mergedeep==1.3.4 +deepdiff==8.6.1 +toml==0.10.2 +sympy==1.14.0 +hf_transfer==0.1.9 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +soupsieve==2.8.1 +jsonlines==4.0.0 +xxhash==3.6.0 +pyzmq==27.1.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +omegaconf==2.3.0 +Werkzeug==3.1.4 +urllib3==2.6.2 +fsspec==2025.9.0 +pillow==12.0.0 +pytorch-triton==3.1.0+cf34004b8a +cuda-pathfinder==1.2.2 +gdown==5.2.0 +beautifulsoup4==4.14.3 +mypy_extensions==1.1.0 +numba==0.63.1 +cmake==4.1.3 +certifi==2025.11.12 +num2words==0.5.14 +fasteners==0.20 +python-xlib==0.33 +nvidia-nvjitlink-cu12==12.6.85 +platformdirs==4.5.1 +torch==2.9.1+cu130 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-nvshmem-cu12==3.4.5 +ImageIO==2.37.2 +termcolor==3.2.0 +zarr==2.18.3 +six==1.17.0 +blinker==1.9.0 +wandb==0.21.4 +MarkupSafe==3.0.3 +wheel==0.45.1 +dill==0.4.0 +typing-inspect==0.9.0 +cuda-bindings==12.9.4 +pip==25.3 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +nvidia-cublas-cu12==12.6.4.1 +docopt==0.6.2 +llvmlite==0.46.0 +nvidia-cufft-cu12==11.3.0.4 +opencv-python-headless==4.12.0.88 diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/wandb-metadata.json b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5fbb40095277de637f3be51f1d6e09ebad9ea133 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/wandb-metadata.json @@ -0,0 +1,48 @@ +{ + "os": "Linux-6.8.0-90-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2025-12-31T05:27:55.808088Z", + "args": [ + "--policy.type=diffusion", + "--dataset.repo_id=kabilanKB/lerobotact-dataset", + "--dataset.video_backend=pyav", + "--policy.repo_id=kabilanKB/diffusion-lerobotact", + "--policy.push_to_hub=true", + "--batch_size=64", + "--steps=20000", + "--output_dir=outputs/train/lerobot_diffusion_model", + "--policy.device=cuda", + "--policy.use_amp=true", + "--wandb.enable=true" + ], + "program": "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", + "codePath": "miniconda3/envs/lerobot/bin/lerobot-train", + "codePathLocal": "miniconda3/envs/lerobot/bin/lerobot-train", + "root": "outputs/train/lerobot_diffusion_model", + "host": "zeux", + "executable": "/home/zeux/miniconda3/envs/lerobot/bin/python3.10", + "cpu_count": 24, + "cpu_count_logical": 24, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "gpu_count": 1, + "disk": { + "/": { + "total": "200449576960", + "used": "34453942272" + } + }, + "memory": { + "total": "134489886720" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e" + } + ], + "cudaVersion": "13.0", + "writerId": "io3tzplzowclvgx723akjha2ri4r13tq" +} \ No newline at end of file diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/wandb-summary.json b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..50f47f488bb08ecb10cc44bb480d2f7aecee2421 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/dataloading_s":0.24293268202600302,"train/update_s":0.05707469244138338,"train/lr":8.586631418937904e-09,"_wandb":{"runtime":6120},"_runtime":6120.227700728,"_step":20000,"_timestamp":1.7671647922038405e+09,"train/episodes":1440.3060650388206,"train/epochs":72.01530325194103,"train/loss":0.004200962032773532,"train/steps":20000,"train/samples":1280000,"train/grad_norm":0.12352929046377539} \ No newline at end of file diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-core.log b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1d697700cd51afdee987ebfd6021182ced507049 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2025-12-31T10:57:55.817410779+05:30","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprjepb5tw/port-443105.txt","pid":443105,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-12-31T10:57:55.817712252+05:30","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-443105-443164-363025454/socket","Net":"unix"}} +{"time":"2025-12-31T10:57:55.817816929+05:30","level":"INFO","msg":"server: will exit if parent process dies","ppid":443105} +{"time":"2025-12-31T10:57:56.012927585+05:30","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-12-31T10:57:56.01778151+05:30","level":"INFO","msg":"handleInformInit: received","streamId":"pkxhwzf2","id":"1(@)"} +{"time":"2025-12-31T10:57:56.434105248+05:30","level":"INFO","msg":"handleInformInit: stream started","streamId":"pkxhwzf2","id":"1(@)"} +{"time":"2025-12-31T12:39:57.174936104+05:30","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-12-31T12:39:57.174978069+05:30","level":"INFO","msg":"server is shutting down"} +{"time":"2025-12-31T12:39:57.174983505+05:30","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-12-31T12:39:57.175021459+05:30","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-443105-443164-363025454/socket","Net":"unix"}} +{"time":"2025-12-31T12:39:57.175048902+05:30","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-12-31T12:39:57.432154481+05:30","level":"ERROR","msg":"processOutgoingData: flush error","error":"write unix /tmp/wandb-443105-443164-363025454/socket->@: use of closed network connection","id":"1(@)"} +{"time":"2025-12-31T12:39:59.500264553+05:30","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-12-31T12:39:59.500277488+05:30","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-12-31T12:39:59.500282582+05:30","level":"INFO","msg":"server is closed"} diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-internal.log b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..435ff0e7547bc21b79e4668fb4e04f9cc575ef78 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-12-31T10:57:56.017911213+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-31T10:57:56.434086163+05:30","level":"INFO","msg":"stream: created new stream","id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.434103309+05:30","level":"INFO","msg":"stream: started","id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.434140193+05:30","level":"INFO","msg":"writer: started","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.434285297+05:30","level":"INFO","msg":"handler: started","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T10:57:56.43430051+05:30","level":"INFO","msg":"sender: started","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:57.174973936+05:30","level":"INFO","msg":"stream: closing","id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:59.116638777+05:30","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-31T12:39:59.491193968+05:30","level":"INFO","msg":"handler: closed","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:59.491242199+05:30","level":"INFO","msg":"sender: closed","stream_id":"pkxhwzf2"} +{"time":"2025-12-31T12:39:59.491246801+05:30","level":"INFO","msg":"stream: closed","id":"pkxhwzf2"} diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug.log b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1deace37fc1132464dcd31b39e80fe83b458e5c5 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug.log @@ -0,0 +1,23 @@ +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Configure stats pid to 443105 +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug.log +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/logs/debug-internal.log +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:init():813] calling init triggers +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'kabilanKB/lerobotact-dataset', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'kabilanKB/diffusion-lerobotact', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'horizon': 16, 'n_action_steps': 8, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet18', 'crop_shape': [84, 84], 'crop_is_random': True, 'pretrained_backbone_weights': None, 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [512, 1024, 2048], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDPM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': None, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500}, 'output_dir': 'outputs/train/lerobot_diffusion_model', 'job_name': 'diffusion', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 64, 'steps': 20000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-31 10:57:55,808 INFO MainThread:443105 [wandb_init.py:init():854] starting backend +2025-12-31 10:57:56,013 INFO MainThread:443105 [wandb_init.py:init():857] sending inform_init request +2025-12-31 10:57:56,014 INFO MainThread:443105 [wandb_init.py:init():865] backend started and connected +2025-12-31 10:57:56,015 INFO MainThread:443105 [wandb_init.py:init():936] updated telemetry +2025-12-31 10:57:56,016 INFO MainThread:443105 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-31 10:57:56,946 INFO MainThread:443105 [wandb_init.py:init():1011] starting run threads in backend +2025-12-31 10:57:57,005 INFO MainThread:443105 [wandb_run.py:_console_start():2506] atexit reg +2025-12-31 10:57:57,005 INFO MainThread:443105 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-31 10:57:57,006 INFO MainThread:443105 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-31 10:57:57,006 INFO MainThread:443105 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-31 10:57:57,006 INFO MainThread:443105 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-31 12:39:57,174 INFO wandb-AsyncioManager-main:443105 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-31 12:39:57,174 INFO wandb-AsyncioManager-main:443105 [mailbox.py:close():137] Closing mailbox, abandoning 2 handles. diff --git a/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/run-pkxhwzf2.wandb b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/run-pkxhwzf2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..12de9fd476bdf7a8a0e1e521473379ea981c3f80 --- /dev/null +++ b/lerobot_diffusion_model/wandb/run-20251231_105755-pkxhwzf2/run-pkxhwzf2.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3032686bd6fbadc2e8051398a274c06a646f3a074f155e769434cbab4ab6ec36 +size 1383097 diff --git a/lerobotact_so101_test/checkpoints/020000/pretrained_model/config.json b/lerobotact_so101_test/checkpoints/020000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..79fe9d110b15fa667cf70241abe27744ae00ae7e --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/pretrained_model/config.json @@ -0,0 +1,62 @@ +{ + "type": "act", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "local/lerobotact-so101-model", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 100, + "n_action_steps": 100, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/lerobotact_so101_test/checkpoints/020000/pretrained_model/model.safetensors b/lerobotact_so101_test/checkpoints/020000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..da7301a380d01aec90725dd2c6826311b8b23e3a --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5584c90bf5476232a38195723748bdfd2803f267370cd824167ac45fa71efac +size 206699736 diff --git a/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor.json b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..9ccb342474cecbf2bdbb034d27169a351ad96d2a --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..85a57f68a0f378553fb7925dd57f1b6690197ba8 --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a521cd2d101c29b05f1a61273cce28a42771c079c5568672627c9c96c355c7f8 +size 6544 diff --git a/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor.json b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..231cd876eb2d02a9d4bdbb479cdc8b9a71f2ebae --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,56 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..85a57f68a0f378553fb7925dd57f1b6690197ba8 --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a521cd2d101c29b05f1a61273cce28a42771c079c5568672627c9c96c355c7f8 +size 6544 diff --git a/lerobotact_so101_test/checkpoints/020000/pretrained_model/train_config.json b/lerobotact_so101_test/checkpoints/020000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..58469260ad4596222bf04dc74d4eb854ec8e5f72 --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/pretrained_model/train_config.json @@ -0,0 +1,185 @@ +{ + "dataset": { + "repo_id": "lerobot-act", + "root": "/home/zeux/lerobot_shadow/lerobotact-dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav", + "streaming": false + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": true, + "push_to_hub": true, + "repo_id": "local/lerobotact-so101-model", + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 100, + "n_action_steps": 100, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": false, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/lerobotact_so101_test", + "job_name": "act", + "resume": false, + "seed": 1000, + "num_workers": 8, + "batch_size": 64, + "steps": 20000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "umsnhs3c", + "mode": null + }, + "checkpoint_path": null, + "rename_map": {} +} \ No newline at end of file diff --git a/lerobotact_so101_test/checkpoints/020000/training_state/optimizer_param_groups.json b/lerobotact_so101_test/checkpoints/020000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/lerobotact_so101_test/checkpoints/020000/training_state/optimizer_state.safetensors b/lerobotact_so101_test/checkpoints/020000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..30c2cb8eea0d13092f292c27aeff2dc32dd7567d --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e52872f041ee88fa55503a14713345edd5e59c0122022d4edd7e09fa9797ef2f +size 412817652 diff --git a/lerobotact_so101_test/checkpoints/020000/training_state/rng_state.safetensors b/lerobotact_so101_test/checkpoints/020000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e129b7ff0ccb3a33cec5ef936a60d50ab90843f1 --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4af395af2035b432cafad4c76b2b5b5bd57605bf11c3debbf4ef81ab68414f77 +size 15708 diff --git a/lerobotact_so101_test/checkpoints/020000/training_state/training_step.json b/lerobotact_so101_test/checkpoints/020000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9bb47026c5d5237ca6fc5dbff6020dd122ea05 --- /dev/null +++ b/lerobotact_so101_test/checkpoints/020000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 20000 +} \ No newline at end of file diff --git a/lerobotact_so101_test/wandb/debug-internal.log b/lerobotact_so101_test/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c0268120a0f96b912e2c8980b20b71bde80ea81b --- /dev/null +++ b/lerobotact_so101_test/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-12-30T22:21:25.067635214+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-30T22:21:25.469149544+05:30","level":"INFO","msg":"stream: created new stream","id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.469169375+05:30","level":"INFO","msg":"stream: started","id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.469678286+05:30","level":"INFO","msg":"sender: started","stream_id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.469715429+05:30","level":"INFO","msg":"writer: started","stream_id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.46971529+05:30","level":"INFO","msg":"handler: started","stream_id":"umsnhs3c"} +{"time":"2025-12-30T23:24:27.285997331+05:30","level":"INFO","msg":"stream: closing","id":"umsnhs3c"} +{"time":"2025-12-30T23:26:13.048119992+05:30","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-30T23:26:13.404350789+05:30","level":"INFO","msg":"handler: closed","stream_id":"umsnhs3c"} +{"time":"2025-12-30T23:26:13.404441244+05:30","level":"INFO","msg":"sender: closed","stream_id":"umsnhs3c"} +{"time":"2025-12-30T23:26:13.404451494+05:30","level":"INFO","msg":"stream: closed","id":"umsnhs3c"} diff --git a/lerobotact_so101_test/wandb/debug.log b/lerobotact_so101_test/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4d43860922864453ef20a46e6246bc02854c8eab --- /dev/null +++ b/lerobotact_so101_test/wandb/debug.log @@ -0,0 +1,23 @@ +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Configure stats pid to 2366462 +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug.log +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-internal.log +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:init():813] calling init triggers +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'lerobot-act', 'root': '/home/zeux/lerobot_shadow/lerobotact-dataset', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'local/lerobotact-so101-model', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'chunk_size': 100, 'n_action_steps': 100, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/lerobotact_so101_test', 'job_name': 'act', 'resume': False, 'seed': 1000, 'num_workers': 8, 'batch_size': 64, 'steps': 20000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:init():854] starting backend +2025-12-30 22:21:25,063 INFO MainThread:2366462 [wandb_init.py:init():857] sending inform_init request +2025-12-30 22:21:25,065 INFO MainThread:2366462 [wandb_init.py:init():865] backend started and connected +2025-12-30 22:21:25,066 INFO MainThread:2366462 [wandb_init.py:init():936] updated telemetry +2025-12-30 22:21:25,067 INFO MainThread:2366462 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-30 22:21:25,942 INFO MainThread:2366462 [wandb_init.py:init():1011] starting run threads in backend +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_console_start():2506] atexit reg +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-30 23:24:27,286 INFO wandb-AsyncioManager-main:2366462 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-30 23:24:27,286 INFO wandb-AsyncioManager-main:2366462 [mailbox.py:close():137] Closing mailbox, abandoning 2 handles. diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/config.yaml b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5d0f6914cb505d2003b6dd93484cda896a363c7 --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/config.yaml @@ -0,0 +1,227 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + uohrn0r1p3vw19t50q4pmjdybj2b6u53: + args: + - --dataset.repo_id=lerobot-act + - --dataset.root=/home/zeux/lerobot_shadow/lerobotact-dataset + - --dataset.video_backend=pyav + - --policy.type=act + - --policy.repo_id=local/lerobotact-so101-model + - --output_dir=outputs/train/lerobotact_so101_test + - --policy.device=cuda + - --policy.use_amp=true + - --batch_size=64 + - --num_workers=8 + - --steps=20000 + - --wandb.enable=true + - --policy.push_to_hub=true + codePath: miniconda3/envs/lerobot/bin/lerobot-train + codePathLocal: miniconda3/envs/lerobot/bin/lerobot-train + cpu_count: 24 + cpu_count_logical: 24 + cudaVersion: "13.0" + disk: + /: + total: "200449576960" + used: "34402750464" + executable: /home/zeux/miniconda3/envs/lerobot/bin/python3.10 + gpu: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + gpu_count: 1 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Workstation Edition + uuid: GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e + host: zeux + memory: + total: "134489886720" + os: Linux-6.8.0-90-generic-x86_64-with-glibc2.35 + program: /home/zeux/miniconda3/envs/lerobot/bin/lerobot-train + python: CPython 3.10.19 + root: outputs/train/lerobotact_so101_test + startedAt: "2025-12-30T16:51:24.859247Z" + writerId: uohrn0r1p3vw19t50q4pmjdybj2b6u53 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + "3": + - 13 + - 15 + - 16 + - 61 + "4": 3.10.19 + "5": 0.21.4 + "6": 4.57.3 + "10": + - 21 + "12": 0.21.4 + "13": linux-x86_64 +batch_size: + value: 64 +checkpoint_path: + value: null +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: lerobot-act + revision: null + root: /home/zeux/lerobot_shadow/lerobotact-dataset + streaming: false + use_imagenet_stats: true + video_backend: pyav +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: act +log_freq: + value: 200 +num_workers: + value: 8 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 1e-05 + type: adamw + weight_decay: 0.0001 +output_dir: + value: outputs/train/lerobotact_so101_test +policy: + value: + chunk_size: 100 + device: cuda + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + kl_weight: 10 + latent_dim: 32 + license: null + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: MEAN_STD + optimizer_lr: 1e-05 + optimizer_lr_backbone: 1e-05 + optimizer_weight_decay: 0.0001 + pre_norm: false + pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1 + pretrained_path: null + private: null + push_to_hub: true + replace_final_stride_with_dilation: false + repo_id: local/lerobotact-so101-model + tags: null + temporal_ensemble_coeff: null + type: act + use_amp: true + use_vae: true + vision_backbone: resnet18 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 20000 +scheduler: + value: null +seed: + value: 1000 +steps: + value: 20000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/output.log b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4a3fe8bc58c8ee45c96ab5383b4a82d6677151ae --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/output.log @@ -0,0 +1,181 @@ +INFO 2025-12-30 22:21:26 db_utils.py:102 Logs will be synced with wandb. +INFO 2025-12-30 22:21:26 db_utils.py:103 Track this run --> https://wandb.ai/nwaves-ai-nwaves/lerobot/runs/umsnhs3c +INFO 2025-12-30 22:21:26 ot_train.py:183 Creating dataset +INFO 2025-12-30 22:21:26 ot_train.py:202 Creating policy +INFO 2025-12-30 22:21:26 ot_train.py:247 Creating optimizer and scheduler +INFO 2025-12-30 22:21:26 ot_train.py:259 Output dir: outputs/train/lerobotact_so101_test +INFO 2025-12-30 22:21:26 ot_train.py:264 cfg.steps=20000 (20K) +INFO 2025-12-30 22:21:26 ot_train.py:265 dataset.num_frames=17774 (18K) +INFO 2025-12-30 22:21:26 ot_train.py:266 dataset.num_episodes=20 +INFO 2025-12-30 22:21:26 ot_train.py:269 Effective batch size: 64 x 1 = 64 +INFO 2025-12-30 22:21:26 ot_train.py:270 num_learnable_params=51597190 (52M) +INFO 2025-12-30 22:21:26 ot_train.py:271 num_total_params=51597190 (52M) +INFO 2025-12-30 22:21:26 ot_train.py:327 Start offline training on a fixed dataset +INFO 2025-12-30 22:22:07 ot_train.py:354 step:200 smpl:13K ep:14 epch:0.72 loss:5.385 grdn:78.354 lr:1.0e-05 updt_s:0.187 data_s:0.019 +INFO 2025-12-30 22:22:47 ot_train.py:354 step:400 smpl:26K ep:29 epch:1.44 loss:2.072 grdn:35.598 lr:1.0e-05 updt_s:0.180 data_s:0.017 +INFO 2025-12-30 22:23:24 ot_train.py:354 step:600 smpl:38K ep:43 epch:2.16 loss:1.610 grdn:33.260 lr:1.0e-05 updt_s:0.172 data_s:0.017 +INFO 2025-12-30 22:24:01 ot_train.py:354 step:800 smpl:51K ep:58 epch:2.88 loss:1.287 grdn:29.700 lr:1.0e-05 updt_s:0.173 data_s:0.008 +INFO 2025-12-30 22:24:39 ot_train.py:354 step:1K smpl:64K ep:72 epch:3.60 loss:1.028 grdn:26.807 lr:1.0e-05 updt_s:0.173 data_s:0.016 +INFO 2025-12-30 22:25:17 ot_train.py:354 step:1K smpl:77K ep:86 epch:4.32 loss:0.827 grdn:24.517 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:25:55 ot_train.py:354 step:1K smpl:90K ep:101 epch:5.04 loss:0.666 grdn:22.966 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:26:32 ot_train.py:354 step:2K smpl:102K ep:115 epch:5.76 loss:0.546 grdn:21.008 lr:1.0e-05 updt_s:0.175 data_s:0.008 +INFO 2025-12-30 22:27:10 ot_train.py:354 step:2K smpl:115K ep:130 epch:6.48 loss:0.451 grdn:18.823 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:27:48 ot_train.py:354 step:2K smpl:128K ep:144 epch:7.20 loss:0.377 grdn:18.360 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:28:25 ot_train.py:354 step:2K smpl:141K ep:158 epch:7.92 loss:0.328 grdn:16.874 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:29:03 ot_train.py:354 step:2K smpl:154K ep:173 epch:8.64 loss:0.286 grdn:16.186 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:29:42 ot_train.py:354 step:3K smpl:166K ep:187 epch:9.36 loss:0.256 grdn:15.542 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:30:20 ot_train.py:354 step:3K smpl:179K ep:202 epch:10.08 loss:0.235 grdn:14.292 lr:1.0e-05 updt_s:0.175 data_s:0.016 +INFO 2025-12-30 22:30:56 ot_train.py:354 step:3K smpl:192K ep:216 epch:10.80 loss:0.213 grdn:13.921 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:31:34 ot_train.py:354 step:3K smpl:205K ep:230 epch:11.52 loss:0.202 grdn:14.183 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:32:13 ot_train.py:354 step:3K smpl:218K ep:245 epch:12.24 loss:0.187 grdn:12.758 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:32:49 ot_train.py:354 step:4K smpl:230K ep:259 epch:12.96 loss:0.179 grdn:12.676 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:33:27 ot_train.py:354 step:4K smpl:243K ep:274 epch:13.68 loss:0.167 grdn:12.130 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:34:06 ot_train.py:354 step:4K smpl:256K ep:288 epch:14.40 loss:0.159 grdn:12.191 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:34:44 ot_train.py:354 step:4K smpl:269K ep:302 epch:15.12 loss:0.153 grdn:12.016 lr:1.0e-05 updt_s:0.175 data_s:0.017 +INFO 2025-12-30 22:35:21 ot_train.py:354 step:4K smpl:282K ep:317 epch:15.84 loss:0.146 grdn:11.151 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:35:59 ot_train.py:354 step:5K smpl:294K ep:331 epch:16.56 loss:0.139 grdn:11.220 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:36:37 ot_train.py:354 step:5K smpl:307K ep:346 epch:17.28 loss:0.135 grdn:10.828 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:37:14 ot_train.py:354 step:5K smpl:320K ep:360 epch:18.00 loss:0.129 grdn:10.874 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:37:52 ot_train.py:354 step:5K smpl:333K ep:374 epch:18.72 loss:0.125 grdn:10.897 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:38:30 ot_train.py:354 step:5K smpl:346K ep:389 epch:19.44 loss:0.121 grdn:10.874 lr:1.0e-05 updt_s:0.175 data_s:0.017 +INFO 2025-12-30 22:39:08 ot_train.py:354 step:6K smpl:358K ep:403 epch:20.16 loss:0.118 grdn:10.397 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:39:45 ot_train.py:354 step:6K smpl:371K ep:418 epch:20.88 loss:0.114 grdn:9.897 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:40:23 ot_train.py:354 step:6K smpl:384K ep:432 epch:21.60 loss:0.111 grdn:9.926 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:41:01 ot_train.py:354 step:6K smpl:397K ep:446 epch:22.32 loss:0.110 grdn:10.163 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:41:39 ot_train.py:354 step:6K smpl:410K ep:461 epch:23.04 loss:0.105 grdn:9.610 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:42:16 ot_train.py:354 step:7K smpl:422K ep:475 epch:23.77 loss:0.103 grdn:9.887 lr:1.0e-05 updt_s:0.175 data_s:0.008 +INFO 2025-12-30 22:42:54 ot_train.py:354 step:7K smpl:435K ep:490 epch:24.49 loss:0.101 grdn:9.652 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:43:32 ot_train.py:354 step:7K smpl:448K ep:504 epch:25.21 loss:0.098 grdn:9.175 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:44:09 ot_train.py:354 step:7K smpl:461K ep:519 epch:25.93 loss:0.096 grdn:8.538 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:44:47 ot_train.py:354 step:7K smpl:474K ep:533 epch:26.65 loss:0.095 grdn:9.328 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:45:25 ot_train.py:354 step:8K smpl:486K ep:547 epch:27.37 loss:0.091 grdn:8.435 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:46:04 ot_train.py:354 step:8K smpl:499K ep:562 epch:28.09 loss:0.091 grdn:8.712 lr:1.0e-05 updt_s:0.175 data_s:0.017 +INFO 2025-12-30 22:46:40 ot_train.py:354 step:8K smpl:512K ep:576 epch:28.81 loss:0.088 grdn:8.820 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:47:18 ot_train.py:354 step:8K smpl:525K ep:591 epch:29.53 loss:0.087 grdn:8.367 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:47:56 ot_train.py:354 step:8K smpl:538K ep:605 epch:30.25 loss:0.086 grdn:8.446 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:48:33 ot_train.py:354 step:9K smpl:550K ep:619 epch:30.97 loss:0.084 grdn:8.888 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:49:11 ot_train.py:354 step:9K smpl:563K ep:634 epch:31.69 loss:0.084 grdn:8.413 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:49:49 ot_train.py:354 step:9K smpl:576K ep:648 epch:32.41 loss:0.082 grdn:8.506 lr:1.0e-05 updt_s:0.175 data_s:0.016 +INFO 2025-12-30 22:50:28 ot_train.py:354 step:9K smpl:589K ep:663 epch:33.13 loss:0.080 grdn:8.055 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:51:04 ot_train.py:354 step:9K smpl:602K ep:677 epch:33.85 loss:0.079 grdn:8.137 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:51:42 ot_train.py:354 step:10K smpl:614K ep:691 epch:34.57 loss:0.080 grdn:8.636 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:52:20 ot_train.py:354 step:10K smpl:627K ep:706 epch:35.29 loss:0.077 grdn:7.791 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:52:57 ot_train.py:354 step:10K smpl:640K ep:720 epch:36.01 loss:0.076 grdn:7.933 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:53:35 ot_train.py:354 step:10K smpl:653K ep:735 epch:36.73 loss:0.075 grdn:7.796 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:54:14 ot_train.py:354 step:10K smpl:666K ep:749 epch:37.45 loss:0.075 grdn:8.141 lr:1.0e-05 updt_s:0.175 data_s:0.017 +INFO 2025-12-30 22:54:51 ot_train.py:354 step:11K smpl:678K ep:763 epch:38.17 loss:0.074 grdn:7.763 lr:1.0e-05 updt_s:0.174 data_s:0.015 +INFO 2025-12-30 22:55:28 ot_train.py:354 step:11K smpl:691K ep:778 epch:38.89 loss:0.072 grdn:7.208 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 22:56:06 ot_train.py:354 step:11K smpl:704K ep:792 epch:39.61 loss:0.072 grdn:7.813 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:56:44 ot_train.py:354 step:11K smpl:717K ep:807 epch:40.33 loss:0.070 grdn:7.335 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:57:23 ot_train.py:354 step:11K smpl:730K ep:821 epch:41.05 loss:0.070 grdn:7.385 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:57:59 ot_train.py:354 step:12K smpl:742K ep:835 epch:41.77 loss:0.068 grdn:6.946 lr:1.0e-05 updt_s:0.175 data_s:0.008 +INFO 2025-12-30 22:58:37 ot_train.py:354 step:12K smpl:755K ep:850 epch:42.49 loss:0.068 grdn:7.068 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 22:59:15 ot_train.py:354 step:12K smpl:768K ep:864 epch:43.21 loss:0.068 grdn:7.043 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 22:59:52 ot_train.py:354 step:12K smpl:781K ep:879 epch:43.93 loss:0.067 grdn:7.570 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:00:30 ot_train.py:354 step:12K smpl:794K ep:893 epch:44.65 loss:0.068 grdn:8.074 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:01:08 ot_train.py:354 step:13K smpl:806K ep:907 epch:45.37 loss:0.065 grdn:6.668 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:01:47 ot_train.py:354 step:13K smpl:819K ep:922 epch:46.09 loss:0.064 grdn:6.845 lr:1.0e-05 updt_s:0.175 data_s:0.017 +INFO 2025-12-30 23:02:23 ot_train.py:354 step:13K smpl:832K ep:936 epch:46.81 loss:0.063 grdn:6.336 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:03:01 ot_train.py:354 step:13K smpl:845K ep:951 epch:47.53 loss:0.065 grdn:7.925 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:03:39 ot_train.py:354 step:13K smpl:858K ep:965 epch:48.25 loss:0.061 grdn:6.012 lr:1.0e-05 updt_s:0.174 data_s:0.015 +INFO 2025-12-30 23:04:16 ot_train.py:354 step:14K smpl:870K ep:979 epch:48.97 loss:0.063 grdn:7.101 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:04:54 ot_train.py:354 step:14K smpl:883K ep:994 epch:49.69 loss:0.062 grdn:6.689 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:05:32 ot_train.py:354 step:14K smpl:896K ep:1K epch:50.41 loss:0.061 grdn:6.310 lr:1.0e-05 updt_s:0.175 data_s:0.017 +INFO 2025-12-30 23:06:11 ot_train.py:354 step:14K smpl:909K ep:1K epch:51.13 loss:0.061 grdn:6.981 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:06:47 ot_train.py:354 step:14K smpl:922K ep:1K epch:51.85 loss:0.060 grdn:6.674 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:07:25 ot_train.py:354 step:15K smpl:934K ep:1K epch:52.57 loss:0.059 grdn:6.552 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:08:04 ot_train.py:354 step:15K smpl:947K ep:1K epch:53.29 loss:0.060 grdn:6.419 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:08:40 ot_train.py:354 step:15K smpl:960K ep:1K epch:54.01 loss:0.059 grdn:6.388 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:09:18 ot_train.py:354 step:15K smpl:973K ep:1K epch:54.73 loss:0.058 grdn:6.228 lr:1.0e-05 updt_s:0.175 data_s:0.016 +INFO 2025-12-30 23:09:57 ot_train.py:354 step:15K smpl:986K ep:1K epch:55.45 loss:0.058 grdn:6.124 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:10:35 ot_train.py:354 step:16K smpl:998K ep:1K epch:56.17 loss:0.057 grdn:6.267 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:11:11 ot_train.py:354 step:16K smpl:1M ep:1K epch:56.89 loss:0.057 grdn:6.132 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:11:50 ot_train.py:354 step:16K smpl:1M ep:1K epch:57.61 loss:0.057 grdn:5.948 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:12:28 ot_train.py:354 step:16K smpl:1M ep:1K epch:58.33 loss:0.056 grdn:5.668 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:13:04 ot_train.py:354 step:16K smpl:1M ep:1K epch:59.05 loss:0.056 grdn:6.490 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:13:43 ot_train.py:354 step:17K smpl:1M ep:1K epch:59.77 loss:0.054 grdn:5.572 lr:1.0e-05 updt_s:0.175 data_s:0.016 +INFO 2025-12-30 23:14:21 ot_train.py:354 step:17K smpl:1M ep:1K epch:60.49 loss:0.055 grdn:5.762 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:14:59 ot_train.py:354 step:17K smpl:1M ep:1K epch:61.21 loss:0.054 grdn:5.813 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:15:36 ot_train.py:354 step:17K smpl:1M ep:1K epch:61.93 loss:0.055 grdn:6.122 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:16:14 ot_train.py:354 step:17K smpl:1M ep:1K epch:62.65 loss:0.053 grdn:5.502 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:16:52 ot_train.py:354 step:18K smpl:1M ep:1K epch:63.37 loss:0.053 grdn:5.398 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:17:31 ot_train.py:354 step:18K smpl:1M ep:1K epch:64.09 loss:0.052 grdn:5.766 lr:1.0e-05 updt_s:0.175 data_s:0.017 +INFO 2025-12-30 23:18:07 ot_train.py:354 step:18K smpl:1M ep:1K epch:64.81 loss:0.052 grdn:5.474 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:18:45 ot_train.py:354 step:18K smpl:1M ep:1K epch:65.53 loss:0.052 grdn:5.879 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:19:24 ot_train.py:354 step:18K smpl:1M ep:1K epch:66.25 loss:0.053 grdn:6.055 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:20:00 ot_train.py:354 step:19K smpl:1M ep:1K epch:66.97 loss:0.051 grdn:5.740 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:20:38 ot_train.py:354 step:19K smpl:1M ep:1K epch:67.69 loss:0.051 grdn:5.505 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:21:17 ot_train.py:354 step:19K smpl:1M ep:1K epch:68.41 loss:0.052 grdn:5.589 lr:1.0e-05 updt_s:0.175 data_s:0.016 +INFO 2025-12-30 23:21:55 ot_train.py:354 step:19K smpl:1M ep:1K epch:69.13 loss:0.051 grdn:5.713 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:22:31 ot_train.py:354 step:19K smpl:1M ep:1K epch:69.85 loss:0.051 grdn:5.628 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:23:10 ot_train.py:354 step:20K smpl:1M ep:1K epch:70.57 loss:0.051 grdn:6.003 lr:1.0e-05 updt_s:0.174 data_s:0.016 +INFO 2025-12-30 23:23:48 ot_train.py:354 step:20K smpl:1M ep:1K epch:71.30 loss:0.050 grdn:5.499 lr:1.0e-05 updt_s:0.174 data_s:0.017 +INFO 2025-12-30 23:24:24 ot_train.py:354 step:20K smpl:1M ep:1K epch:72.02 loss:0.049 grdn:5.096 lr:1.0e-05 updt_s:0.174 data_s:0.008 +INFO 2025-12-30 23:24:24 ot_train.py:364 Checkpoint policy after step 20000 +INFO 2025-12-30 23:24:26 ot_train.py:435 End of training +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 407, in hf_raise_for_status + response.raise_for_status() + File "/home/zeux/.local/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status + raise HTTPError(http_error_msg, response=self) +requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://huggingface.co/api/repos/create + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", line 7, in + sys.exit(main()) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 449, in main + train() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/configs/parser.py", line 233, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 439, in train + unwrapped_policy.push_model_to_hub(cfg) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/policies/pretrained.py", line 211, in push_model_to_hub + repo_id = api.create_repo( + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3779, in create_repo + raise err + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3766, in create_repo + hf_raise_for_status(r) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 471, in hf_raise_for_status + raise _format(HfHubHTTPError, message, response) from e +huggingface_hub.errors.HfHubHTTPError: (Request ID: Root=1-695411d2-5495a95537b60dc259cf6999;1a73185a-24be-4062-83c1-86cb7f3651dd) + +403 Forbidden: You don't have the rights to create a model under the namespace "local". +Cannot access content at: https://huggingface.co/api/repos/create. +Make sure your token has the correct permissions. +Traceback (most recent call last): + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 407, in hf_raise_for_status + response.raise_for_status() + File "/home/zeux/.local/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status + raise HTTPError(http_error_msg, response=self) +requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://huggingface.co/api/repos/create + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", line 7, in + sys.exit(main()) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 449, in main + train() + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/configs/parser.py", line 233, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/scripts/lerobot_train.py", line 439, in train + unwrapped_policy.push_model_to_hub(cfg) + File "/home/zeux/miniconda3/envs/lerobot/lib/python3.10/site-packages/lerobot/policies/pretrained.py", line 211, in push_model_to_hub + repo_id = api.create_repo( + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3779, in create_repo + raise err + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 3766, in create_repo + hf_raise_for_status(r) + File "/home/zeux/.local/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 471, in hf_raise_for_status + raise _format(HfHubHTTPError, message, response) from e +huggingface_hub.errors.HfHubHTTPError: (Request ID: Root=1-695411d2-5495a95537b60dc259cf6999;1a73185a-24be-4062-83c1-86cb7f3651dd) + +403 Forbidden: You don't have the rights to create a model under the namespace "local". +Cannot access content at: https://huggingface.co/api/repos/create. +Make sure your token has the correct permissions. diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/requirements.txt b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..24e53687369669d9994213578166cd6f5de74e67 --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/requirements.txt @@ -0,0 +1,237 @@ +nvidia-cusolver==12.0.3.29 +nvidia-curand==10.4.0.35 +anyio==4.12.0 +yarl==1.22.0 +comfyui-workflow-templates-media-other==0.3.15 +tomlkit==0.13.3 +cloudpickle==3.1.2 +typing_extensions==4.15.0 +sentry-sdk==2.48.0 +semver==3.0.4 +mpmath==1.3.0 +draccus==0.10.0 +multidict==6.7.0 +nvidia-cufile-cu12==1.11.1.6 +tokenizers==0.22.1 +GitPython==3.1.45 +datasets==4.1.1 +PySide6_Essentials==6.10.1 +comfyui-workflow-templates-media-image==0.3.17 +Pygments==2.19.2 +aiohappyeyeballs==2.6.1 +protobuf==6.33.2 +PySide6==6.10.1 +comfyui-workflow-templates-core==0.3.12 +PySide2==5.15.2.1 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +scipy==1.15.3 +frozenlist==1.8.0 +nvidia-nvshmem-cu13==3.3.24 +transformers==4.57.3 +typing-inspection==0.4.2 +einops==0.8.1 +shellingham==1.5.4 +Farama-Notifications==0.0.4 +psutil==7.1.3 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +nvidia-cufft==12.0.0.15 +cookiecutter==2.6.0 +rich==14.2.0 +SQLAlchemy==2.0.44 +pydantic_core==2.41.5 +nvidia-nvjitlink==13.0.39 +questionary==2.1.1 +requests==2.32.5 +nvidia-cuda-nvrtc==13.0.48 +torchcodec==0.5 +PyOpenGL-accelerate==3.1.10 +comfyui_frontend_package==1.33.10 +networkx==3.4.2 +nvidia-nvtx==13.0.39 +setuptools==80.9.0 +trampoline==0.1.2 +pandas==2.3.3 +httpx==0.28.1 +av==15.1.0 +comfy-cli==1.5.3 +PyOpenGL==3.1.10 +tomli==2.3.0 +safetensors==0.7.0 +regex==2025.11.3 +wcwidth==0.2.14 +pyyaml-include==1.4.1 +gymnasium==1.2.3 +kornia==0.8.2 +packaging==25.0 +nvidia-nccl-cu13==2.27.7 +pydantic==2.12.5 +mixpanel==5.0.0 +markdown-it-py==4.0.0 +multiprocess==0.70.16 +usd-exchange==2.1.0 +h11==0.16.0 +mergedeep==1.3.4 +shiboken6==6.10.1 +text-unidecode==1.3 +deepdiff==8.6.1 +toml==0.10.2 +aiosignal==1.4.0 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +uv==0.9.16 +jsonlines==4.0.0 +xxhash==3.6.0 +numpy==2.2.6 +nvidia-cuda-runtime==13.0.48 +arrow==1.4.0 +spandrel==0.4.1 +nvidia-cuda-cupti==13.0.48 +python-dotenv==1.2.1 +async-timeout==5.0.1 +urllib3==2.6.2 +fsspec==2025.9.0 +nvidia-cusparse==12.6.2.49 +binaryornot==0.4.4 +nvidia-cufile==1.15.0.42 +mdurl==0.1.2 +pydantic-settings==2.12.0 +gitdb==4.0.12 +kornia_rs==0.1.10 +attrs==25.4.0 +mypy_extensions==1.1.0 +tqdm==4.67.1 +alembic==1.17.2 +comfyui-embedded-docs==0.3.1 +python-slugify==8.0.4 +cmake==4.1.3 +smmap==5.0.2 +comfyui-workflow-templates-media-api==0.3.14 +torchsde==0.2.6 +pathspec==0.12.1 +sentencepiece==0.2.1 +charset-normalizer==3.4.4 +nvidia-cusparselt-cu13==0.8.0 +python-xlib==0.33 +ruff==0.14.8 +platformdirs==4.5.1 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-cublas==13.0.0.19 +prompt_toolkit==3.0.52 +ImageIO==2.37.2 +termcolor==3.3.0 +wandb==0.21.4 +annotated-types==0.7.0 +comfyui-workflow-templates-media-video==0.3.12 +typer==0.20.0 +dill==0.4.0 +propcache==0.4.1 +typing-inspect==0.9.0 +greenlet==3.3.0 +hf-xet==1.2.0 +shiboken2==5.15.2.1 +tzdata==2025.2 +aiohttp==3.13.2 +websocket-client==1.9.0 +nvidia-cudnn-cu13==9.13.0.50 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +PySide6_Addons==6.10.1 +asgiref==3.11.0 +httpcore==1.0.9 +exceptiongroup==1.3.1 +filelock==3.20.0 +comfyui_workflow_templates==0.7.25 +opencv-python-headless==4.12.0.88 +nvidia-curand-cu12==10.3.7.77 +nvidia-cudnn-cu12==9.5.1.17 +nvidia-nvtx-cu12==12.6.77 +cloudpickle==3.1.2 +pytz==2025.2 +nvidia-cusparselt-cu12==0.6.3 +nvidia-cusparselt-cu12==0.7.1 +sentry-sdk==2.48.0 +importlib_metadata==8.7.1 +draccus==0.10.0 +nvidia-cufile-cu12==1.11.1.6 +datasets==4.1.1 +nvidia-cusolver-cu12==11.7.1.2 +protobuf==6.33.2 +pyserial==3.5 +accelerate==1.12.0 +pfzy==0.3.4 +setuptools==80.9.0 +nvidia-cuda-cupti-cu12==12.6.80 +Farama-Notifications==0.0.4 +rerun-sdk==0.26.2 +PyYAML==6.0.3 +torchvision==0.24.1+cu130 +requests==2.32.5 +torchcodec==0.5 +idna==3.11 +zipp==3.23.0 +pandas==2.3.3 +av==15.1.0 +nvidia-cusparse-cu12==12.5.4.2 +pyyaml-include==1.4.1 +gymnasium==1.2.3 +nvidia-cuda-runtime-cu12==12.6.77 +triton==3.6.0+git9844da95 +triton==3.5.1 +multiprocess==0.70.16 +lerobot==0.4.2 +Jinja2==3.1.6 +click==8.3.1 +nvidia-nccl-cu12==2.28.9 +nvidia-nccl-cu12==2.26.2 +mergedeep==1.3.4 +deepdiff==8.6.1 +toml==0.10.2 +sympy==1.14.0 +hf_transfer==0.1.9 +imageio-ffmpeg==0.6.0 +python-dateutil==2.9.0.post0 +jsonlines==4.0.0 +xxhash==3.6.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +urllib3==2.6.2 +fsspec==2025.9.0 +pillow==12.0.0 +pytorch-triton==3.1.0+cf34004b8a +cuda-pathfinder==1.2.2 +mypy_extensions==1.1.0 +cmake==4.1.3 +certifi==2025.11.12 +num2words==0.5.14 +python-xlib==0.33 +nvidia-nvjitlink-cu12==12.6.85 +platformdirs==4.5.1 +torch==2.9.1+cu130 +inquirerpy==0.3.4 +evdev==1.9.2 +huggingface-hub==0.35.3 +nvidia-nvshmem-cu12==3.4.5 +ImageIO==2.37.2 +termcolor==3.2.0 +six==1.17.0 +wandb==0.21.4 +MarkupSafe==3.0.3 +wheel==0.45.1 +dill==0.4.0 +typing-inspect==0.9.0 +cuda-bindings==12.9.4 +pip==25.3 +pyarrow==22.0.0 +orderly-set==5.5.0 +pynput==1.8.1 +diffusers==0.35.2 +nvidia-cublas-cu12==12.6.4.1 +docopt==0.6.2 +nvidia-cufft-cu12==11.3.0.4 +opencv-python-headless==4.12.0.88 diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/wandb-metadata.json b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1813a13f51e889b3d8d867411e92b64c93a593aa --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/wandb-metadata.json @@ -0,0 +1,50 @@ +{ + "os": "Linux-6.8.0-90-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.19", + "startedAt": "2025-12-30T16:51:24.859247Z", + "args": [ + "--dataset.repo_id=lerobot-act", + "--dataset.root=/home/zeux/lerobot_shadow/lerobotact-dataset", + "--dataset.video_backend=pyav", + "--policy.type=act", + "--policy.repo_id=local/lerobotact-so101-model", + "--output_dir=outputs/train/lerobotact_so101_test", + "--policy.device=cuda", + "--policy.use_amp=true", + "--batch_size=64", + "--num_workers=8", + "--steps=20000", + "--wandb.enable=true", + "--policy.push_to_hub=true" + ], + "program": "/home/zeux/miniconda3/envs/lerobot/bin/lerobot-train", + "codePath": "miniconda3/envs/lerobot/bin/lerobot-train", + "codePathLocal": "miniconda3/envs/lerobot/bin/lerobot-train", + "root": "outputs/train/lerobotact_so101_test", + "host": "zeux", + "executable": "/home/zeux/miniconda3/envs/lerobot/bin/python3.10", + "cpu_count": 24, + "cpu_count_logical": 24, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "gpu_count": 1, + "disk": { + "/": { + "total": "200449576960", + "used": "34402750464" + } + }, + "memory": { + "total": "134489886720" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-8c66f3c4-00c8-00a6-e5ff-633c5558ec8e" + } + ], + "cudaVersion": "13.0", + "writerId": "uohrn0r1p3vw19t50q4pmjdybj2b6u53" +} \ No newline at end of file diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/wandb-summary.json b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..cc66794d1ac7e8dbc746016dcee4a543cfeb37f9 --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":3781},"train/epochs":72.01530325194103,"train/grad_norm":5.096393461227417,"train/loss":0.04866646101698279,"train/kld_loss":0.00021234946325421333,"train/l1_loss":0.04688161984086037,"train/update_s":0.17421317377964443,"train/steps":20000,"train/lr":1.0000000000000021e-05,"_timestamp":1.7671172649994552e+09,"_runtime":3781.343336612,"train/dataloading_s":0.008423557524656644,"train/episodes":1440.3060650388206,"train/samples":1280000,"_step":20000} \ No newline at end of file diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-core.log b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..71c1afe9cc454750c5d42cd63664fa8637c7467e --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2025-12-30T22:21:24.868125706+05:30","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmplygafqt3/port-2366462.txt","pid":2366462,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-12-30T22:21:24.868444382+05:30","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2366462-2366557-826121440/socket","Net":"unix"}} +{"time":"2025-12-30T22:21:24.868491621+05:30","level":"INFO","msg":"server: will exit if parent process dies","ppid":2366462} +{"time":"2025-12-30T22:21:25.063280467+05:30","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-12-30T22:21:25.067590254+05:30","level":"INFO","msg":"handleInformInit: received","streamId":"umsnhs3c","id":"1(@)"} +{"time":"2025-12-30T22:21:25.469171532+05:30","level":"INFO","msg":"handleInformInit: stream started","streamId":"umsnhs3c","id":"1(@)"} +{"time":"2025-12-30T23:24:27.28597383+05:30","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-12-30T23:24:27.286036479+05:30","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-12-30T23:24:27.286052501+05:30","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-12-30T23:24:27.286054627+05:30","level":"INFO","msg":"server is shutting down"} +{"time":"2025-12-30T23:24:27.286078552+05:30","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2366462-2366557-826121440/socket","Net":"unix"}} +{"time":"2025-12-30T23:26:12.997303253+05:30","level":"ERROR","msg":"processOutgoingData: flush error","error":"write unix /tmp/wandb-2366462-2366557-826121440/socket->@: use of closed network connection","id":"1(@)"} +{"time":"2025-12-30T23:26:13.413732199+05:30","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-12-30T23:26:13.41374726+05:30","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-12-30T23:26:13.413752622+05:30","level":"INFO","msg":"server is closed"} diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-internal.log b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c0268120a0f96b912e2c8980b20b71bde80ea81b --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-12-30T22:21:25.067635214+05:30","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-12-30T22:21:25.469149544+05:30","level":"INFO","msg":"stream: created new stream","id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.469169375+05:30","level":"INFO","msg":"stream: started","id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.469678286+05:30","level":"INFO","msg":"sender: started","stream_id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.469715429+05:30","level":"INFO","msg":"writer: started","stream_id":"umsnhs3c"} +{"time":"2025-12-30T22:21:25.46971529+05:30","level":"INFO","msg":"handler: started","stream_id":"umsnhs3c"} +{"time":"2025-12-30T23:24:27.285997331+05:30","level":"INFO","msg":"stream: closing","id":"umsnhs3c"} +{"time":"2025-12-30T23:26:13.048119992+05:30","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-30T23:26:13.404350789+05:30","level":"INFO","msg":"handler: closed","stream_id":"umsnhs3c"} +{"time":"2025-12-30T23:26:13.404441244+05:30","level":"INFO","msg":"sender: closed","stream_id":"umsnhs3c"} +{"time":"2025-12-30T23:26:13.404451494+05:30","level":"INFO","msg":"stream: closed","id":"umsnhs3c"} diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug.log b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4d43860922864453ef20a46e6246bc02854c8eab --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug.log @@ -0,0 +1,23 @@ +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Current SDK version is 0.21.4 +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Configure stats pid to 2366462 +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/.config/wandb/settings +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Loading settings from /home/zeux/wandb/settings +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:setup_run_log_directory():686] Logging user logs to outputs/train/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug.log +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to outputs/train/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/logs/debug-internal.log +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:init():813] calling init triggers +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:init():818] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'lerobot-act', 'root': '/home/zeux/lerobot_shadow/lerobotact-dataset', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav', 'streaming': False}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': True, 'push_to_hub': True, 'repo_id': 'local/lerobotact-so101-model', 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'chunk_size': 100, 'n_action_steps': 100, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/lerobotact_so101_test', 'job_name': 'act', 'resume': False, 'seed': 1000, 'num_workers': 8, 'batch_size': 64, 'steps': 20000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, 'checkpoint_path': None, 'rename_map': {}, '_wandb': {}} +2025-12-30 22:21:24,859 INFO MainThread:2366462 [wandb_init.py:init():854] starting backend +2025-12-30 22:21:25,063 INFO MainThread:2366462 [wandb_init.py:init():857] sending inform_init request +2025-12-30 22:21:25,065 INFO MainThread:2366462 [wandb_init.py:init():865] backend started and connected +2025-12-30 22:21:25,066 INFO MainThread:2366462 [wandb_init.py:init():936] updated telemetry +2025-12-30 22:21:25,067 INFO MainThread:2366462 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout +2025-12-30 22:21:25,942 INFO MainThread:2366462 [wandb_init.py:init():1011] starting run threads in backend +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_console_start():2506] atexit reg +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_redirect():2354] redirect: wrap_raw +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_redirect():2423] Wrapping output streams. +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_run.py:_redirect():2446] Redirects installed. +2025-12-30 22:21:26,000 INFO MainThread:2366462 [wandb_init.py:init():1049] run started, returning control to user process +2025-12-30 23:24:27,286 INFO wandb-AsyncioManager-main:2366462 [service_client.py:_forward_responses():84] Reached EOF. +2025-12-30 23:24:27,286 INFO wandb-AsyncioManager-main:2366462 [mailbox.py:close():137] Closing mailbox, abandoning 2 handles. diff --git a/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/run-umsnhs3c.wandb b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/run-umsnhs3c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..934016530ab593d798a33101b7ec5a33fd24bf4d --- /dev/null +++ b/lerobotact_so101_test/wandb/run-20251230_222124-umsnhs3c/run-umsnhs3c.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24068ccdab525fb73266ed95d8c007a19b34cf6ef0d105edfa55865d59e899dc +size 343707