Delete wandb

wandb/debug-internal.log

@@ -1,12 +0,0 @@
-{"time":"2025-09-10T06:17:54.318252724Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
-{"time":"2025-09-10T06:17:54.493560254Z","level":"INFO","msg":"stream: created new stream","id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493583685Z","level":"INFO","msg":"stream: started","id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493606835Z","level":"INFO","msg":"writer: Do: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493615595Z","level":"INFO","msg":"handler: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493633765Z","level":"INFO","msg":"sender: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.545468692Z","level":"INFO","msg":"stream: closing","id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.78249679Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-09-10T13:36:46.884073306Z","level":"INFO","msg":"handler: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884106596Z","level":"INFO","msg":"sender: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884095846Z","level":"INFO","msg":"writer: Close: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884184498Z","level":"INFO","msg":"stream: closed","id":"od103fqv"}

wandb/debug.log

@@ -1,22 +0,0 @@
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Configure stats pid to 2168
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/projects/policy/wandb/settings
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7/wandb/run-20250910_061753-od103fqv/logs/debug.log
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7/wandb/run-20250910_061753-od103fqv/logs/debug-internal.log
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:init():830] calling init triggers
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
-config: {'dataset': {'repo_id': 'black_table_insert_rope_downsampled_v7', 'root': None, 'episodes': None, 'image_transforms': {'enable': True, 'max_num_transforms': 3, 'random_order': True, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'perspective': {'weight': 1.0, 'type': 'RandomPerspective', 'kwargs': {'distortion_scale': 0.025, 'p': 1.0}}, 'rotation': {'weight': 1.0, 'type': 'RandomRotation', 'kwargs': {'degrees': [-5, 5]}}, 'crop': {'weight': 1.0, 'type': 'RandomCrop', 'kwargs': {'size': [110, 202]}}}, 'downsample': {'resize': {'weight': 1.0, 'type': 'Resize', 'kwargs': {'size': [120, 212]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'relative_actions': True, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [6]}, 'observation.image2': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}, 'observation.image': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}, 'observation.image3': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'optimizer_grad_clip_norm': 10.0, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': True, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': 0, 'pad_language_to': 'max_length', 'num_expert_layers': 0, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0}, 'output_dir': 'outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7', 'job_name': '2025-09-10_06-17-38_svla_rel_v7', 'resume': False, 'seed': 100000, 'num_workers': 4, 'batch_size': 128, 'steps': 20000, 'eval_freq': 2000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 1000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-10, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 0.0001, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
-2025-09-10 06:17:53,893 INFO    MainThread:2168 [wandb_init.py:init():871] starting backend
-2025-09-10 06:17:54,308 INFO    MainThread:2168 [wandb_init.py:init():874] sending inform_init request
-2025-09-10 06:17:54,312 INFO    MainThread:2168 [wandb_init.py:init():882] backend started and connected
-2025-09-10 06:17:54,314 INFO    MainThread:2168 [wandb_init.py:init():953] updated telemetry
-2025-09-10 06:17:54,318 INFO    MainThread:2168 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
-2025-09-10 06:17:54,631 INFO    MainThread:2168 [wandb_init.py:init():1029] starting run threads in backend
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_console_start():2458] atexit reg
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2306] redirect: wrap_raw
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2375] Wrapping output streams.
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2398] Redirects installed.
-2025-09-10 06:17:54,799 INFO    MainThread:2168 [wandb_init.py:init():1075] run started, returning control to user process
-2025-09-10 13:36:46,544 INFO    MsgRouterThr:2168 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

wandb/latest-run/files/config.yaml

@@ -1,280 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.21.0
-        e:
-            k282cdjuxylro9nzqrdyy1qm8gwgxyoa:
-                args:
-                    - --policy.path=lerobot/smolvla_base
-                    - --config_path=src/training_cfg/insert_rope/svla_rel.json
-                    - --dataset.repo_id=black_table_insert_rope_downsampled_v7
-                    - --job_name=2025-09-10_06-17-38_svla_rel_v7
-                    - --output_dir=outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-                codePath: lerobot/scripts/train.py
-                codePathLocal: lerobot/scripts/train.py
-                cpu_count: 4
-                cpu_count_logical: 8
-                cudaVersion: "12.8"
-                disk:
-                    /:
-                        total: "1063956480000"
-                        used: "581626609664"
-                executable: /home/ubuntu/miniconda3/envs/lerobot/bin/python
-                git:
-                    commit: cc0d6bb844d05c377b053051ed40114478c3a7c8
-                    remote: git@github.com:SceniX-AI/policy.git
-                gpu: NVIDIA L40S
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Ada
-                      cudaCores: 18176
-                      memoryTotal: "48305799168"
-                      name: NVIDIA L40S
-                      uuid: GPU-9277cabf-a4d3-fab1-cd41-888c1e1f5b3a
-                host: ip-172-31-8-128
-                memory:
-                    total: "66507816960"
-                os: Linux-6.8.0-1031-aws-x86_64-with-glibc2.39
-                program: /home/ubuntu/projects/policy/lerobot/scripts/train.py
-                python: CPython 3.10.13
-                root: outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-                startedAt: "2025-09-10T06:17:53.891577Z"
-                writerId: k282cdjuxylro9nzqrdyy1qm8gwgxyoa
-        m: []
-        python_version: 3.10.13
-        t:
-            "1":
-                - 1
-                - 11
-                - 41
-                - 49
-                - 51
-                - 71
-            "2":
-                - 1
-                - 11
-                - 41
-                - 49
-                - 51
-                - 71
-            "3":
-                - 13
-                - 15
-                - 16
-                - 61
-            "4": 3.10.13
-            "5": 0.21.0
-            "6": 4.53.1
-            "10":
-                - 21
-            "12": 0.21.0
-            "13": linux-x86_64
-batch_size:
-    value: 128
-dataset:
-    value:
-        episodes: null
-        image_transforms:
-            downsample:
-                resize:
-                    kwargs:
-                        size:
-                            - 120
-                            - 212
-                    type: Resize
-                    weight: 1
-            enable: true
-            max_num_transforms: 3
-            random_order: true
-            tfs:
-                brightness:
-                    kwargs:
-                        brightness:
-                            - 0.8
-                            - 1.2
-                    type: ColorJitter
-                    weight: 1
-                contrast:
-                    kwargs:
-                        contrast:
-                            - 0.8
-                            - 1.2
-                    type: ColorJitter
-                    weight: 1
-                crop:
-                    kwargs:
-                        size:
-                            - 110
-                            - 202
-                    type: RandomCrop
-                    weight: 1
-                hue:
-                    kwargs:
-                        hue:
-                            - -0.05
-                            - 0.05
-                    type: ColorJitter
-                    weight: 1
-                perspective:
-                    kwargs:
-                        distortion_scale: 0.025
-                        p: 1
-                    type: RandomPerspective
-                    weight: 1
-                rotation:
-                    kwargs:
-                        degrees:
-                            - -5
-                            - 5
-                    type: RandomRotation
-                    weight: 1
-                saturation:
-                    kwargs:
-                        saturation:
-                            - 0.5
-                            - 1.5
-                    type: ColorJitter
-                    weight: 1
-                sharpness:
-                    kwargs:
-                        sharpness:
-                            - 0.5
-                            - 1.5
-                    type: SharpnessJitter
-                    weight: 1
-        repo_id: black_table_insert_rope_downsampled_v7
-        revision: null
-        root: null
-        use_imagenet_stats: true
-        video_backend: torchcodec
-env:
-    value: null
-eval:
-    value:
-        batch_size: 50
-        n_episodes: 50
-        use_async_envs: false
-eval_freq:
-    value: 2000
-job_name:
-    value: 2025-09-10_06-17-38_svla_rel_v7
-log_freq:
-    value: 200
-num_workers:
-    value: 4
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        eps: 1e-08
-        grad_clip_norm: 10
-        lr: 0.0001
-        type: adamw
-        weight_decay: 1e-10
-output_dir:
-    value: outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-policy:
-    value:
-        adapt_to_pi_aloha: false
-        add_image_special_tokens: false
-        attention_mode: cross_attn
-        chunk_size: 50
-        device: cuda
-        empty_cameras: 0
-        expert_width_multiplier: 0.75
-        freeze_vision_encoder: true
-        input_features:
-            observation.image:
-                shape:
-                    - 3
-                    - 256
-                    - 256
-                type: VISUAL
-            observation.image2:
-                shape:
-                    - 3
-                    - 256
-                    - 256
-                type: VISUAL
-            observation.image3:
-                shape:
-                    - 3
-                    - 256
-                    - 256
-                type: VISUAL
-            observation.state:
-                shape:
-                    - 6
-                type: STATE
-        load_vlm_weights: true
-        max_action_dim: 32
-        max_period: 4
-        max_state_dim: 32
-        min_period: 0.004
-        n_action_steps: 50
-        n_obs_steps: 1
-        normalization_mapping:
-            ACTION: MEAN_STD
-            STATE: MEAN_STD
-            VISUAL: IDENTITY
-        num_expert_layers: 0
-        num_steps: 10
-        num_vlm_layers: 16
-        optimizer_betas:
-            - 0.9
-            - 0.95
-        optimizer_eps: 1e-08
-        optimizer_grad_clip_norm: 10
-        optimizer_lr: 0.0001
-        optimizer_weight_decay: 1e-10
-        output_features:
-            action:
-                shape:
-                    - 6
-                type: ACTION
-        pad_language_to: max_length
-        prefix_length: 0
-        relative_actions: true
-        resize_imgs_with_padding:
-            - 512
-            - 512
-        scheduler_decay_lr: 2.5e-06
-        scheduler_decay_steps: 30000
-        scheduler_warmup_steps: 1000
-        self_attn_every_n_layers: 2
-        tokenizer_max_length: 48
-        train_expert_only: true
-        train_state_proj: true
-        type: smolvla
-        use_amp: false
-        use_cache: true
-        use_delta_joint_actions_aloha: false
-        vlm_model_name: HuggingFaceTB/SmolVLM2-500M-Video-Instruct
-resume:
-    value: false
-save_checkpoint:
-    value: true
-save_freq:
-    value: 1000
-scheduler:
-    value:
-        decay_lr: 2.5e-06
-        num_decay_steps: 30000
-        num_warmup_steps: 1000
-        peak_lr: 0.0001
-        type: cosine_decay_with_warmup
-seed:
-    value: 100000
-steps:
-    value: 20000
-use_policy_training_preset:
-    value: true
-wandb:
-    value:
-        disable_artifact: true
-        enable: true
-        entity: null
-        mode: null
-        notes: null
-        project: lerobot
-        run_id: null

wandb/latest-run/files/output.log

@@ -1,341 +0,0 @@
-Logs will be synced with wandb.
-INFO 2025-09-10 06:17:54 ndb_utils.py:96 Track this run --> https://wandb.ai/ss7050-columbia/lerobot/runs/od103fqv
-INFO 2025-09-10 06:17:54 ts/train.py:260 Creating dataset
-Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 56/56 [00:00<00:00, 291054.55it/s]
-Downloading data: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 56/56 [00:00<00:00, 317406.79files/s]
-Generating train split: 31026 examples [00:00, 368550.14 examples/s]
-INFO 2025-09-10 06:17:55 ts/train.py:272 Creating policy
-INFO 2025-09-10 06:17:55 ts/train.py:275 Using relative actions: True
-You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0.
-Loading  HuggingFaceTB/SmolVLM2-500M-Video-Instruct weights ...
-INFO 2025-09-10 06:17:57 modeling.py:991 We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).
-Reducing the number of VLM layers to 16 ...
-INFO 2025-09-10 06:18:23 ts/train.py:303 Creating optimizer and scheduler
-INFO 2025-09-10 06:18:23 ts/train.py:315 Output dir: outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-INFO 2025-09-10 06:18:23 ts/train.py:318 cfg.steps=20000 (20K)
-INFO 2025-09-10 06:18:23 ts/train.py:319 dataset.num_frames=31026 (31K)
-INFO 2025-09-10 06:18:23 ts/train.py:320 dataset.num_episodes=56
-INFO 2025-09-10 06:18:23 ts/train.py:321 num_learnable_params=99880992 (100M)
-INFO 2025-09-10 06:18:23 ts/train.py:322 num_total_params=450047792 (450M)
-INFO 2025-09-10 06:18:23 ts/train.py:361 Start offline training on a fixed dataset
-INFO 2025-09-10 06:22:48 ts/train.py:394 step:200 smpl:26K ep:46 epch:0.83 loss:1.131 grdn:2.910 lr:1.0e-05 updt_s:1.287 data_s:0.015
-WARNING 2025-09-10 06:22:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:22:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:27:13 ts/train.py:394 step:400 smpl:51K ep:92 epch:1.65 loss:0.316 grdn:2.630 lr:3.0e-05 updt_s:1.288 data_s:0.013
-WARNING 2025-09-10 06:27:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:27:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:31:35 ts/train.py:394 step:600 smpl:77K ep:139 epch:2.48 loss:0.164 grdn:3.080 lr:5.0e-05 updt_s:1.273 data_s:0.013
-WARNING 2025-09-10 06:31:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:31:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:35:58 ts/train.py:394 step:800 smpl:102K ep:185 epch:3.30 loss:0.135 grdn:3.063 lr:7.0e-05 updt_s:1.278 data_s:0.013
-WARNING 2025-09-10 06:35:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:35:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:40:20 ts/train.py:394 step:1K smpl:128K ep:231 epch:4.13 loss:0.119 grdn:2.490 lr:9.0e-05 updt_s:1.267 data_s:0.014
-WARNING 2025-09-10 06:40:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:40:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:40:20 ts/train.py:403 Checkpoint policy after step 1000
-INFO 2025-09-10 06:44:43 ts/train.py:394 step:1K smpl:154K ep:277 epch:4.95 loss:0.106 grdn:1.772 lr:1.0e-04 updt_s:1.280 data_s:0.002
-WARNING 2025-09-10 06:44:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:44:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:49:06 ts/train.py:394 step:1K smpl:179K ep:323 epch:5.78 loss:0.096 grdn:1.347 lr:1.0e-04 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 06:49:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:49:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:53:31 ts/train.py:394 step:2K smpl:205K ep:370 epch:6.60 loss:0.088 grdn:1.062 lr:9.9e-05 updt_s:1.287 data_s:0.013
-WARNING 2025-09-10 06:53:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:53:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:57:54 ts/train.py:394 step:2K smpl:230K ep:416 epch:7.43 loss:0.084 grdn:0.930 lr:9.9e-05 updt_s:1.278 data_s:0.013
-WARNING 2025-09-10 06:57:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:57:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:02:18 ts/train.py:394 step:2K smpl:256K ep:462 epch:8.25 loss:0.079 grdn:0.797 lr:9.9e-05 updt_s:1.282 data_s:0.014
-WARNING 2025-09-10 07:02:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:02:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:02:18 ts/train.py:403 Checkpoint policy after step 2000
-INFO 2025-09-10 07:06:45 ts/train.py:394 step:2K smpl:282K ep:508 epch:9.08 loss:0.075 grdn:0.692 lr:9.9e-05 updt_s:1.286 data_s:0.014
-WARNING 2025-09-10 07:06:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:06:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:11:06 ts/train.py:394 step:2K smpl:307K ep:554 epch:9.90 loss:0.072 grdn:0.628 lr:9.9e-05 updt_s:1.280 data_s:0.002
-WARNING 2025-09-10 07:11:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:11:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:15:26 ts/train.py:394 step:3K smpl:333K ep:601 epch:10.73 loss:0.069 grdn:0.593 lr:9.8e-05 updt_s:1.262 data_s:0.014
-WARNING 2025-09-10 07:15:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:15:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:19:49 ts/train.py:394 step:3K smpl:358K ep:647 epch:11.55 loss:0.067 grdn:0.531 lr:9.8e-05 updt_s:1.276 data_s:0.013
-WARNING 2025-09-10 07:19:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:19:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:24:13 ts/train.py:394 step:3K smpl:384K ep:693 epch:12.38 loss:0.065 grdn:0.509 lr:9.8e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 07:24:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:24:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:24:13 ts/train.py:403 Checkpoint policy after step 3000
-INFO 2025-09-10 07:28:38 ts/train.py:394 step:3K smpl:410K ep:739 epch:13.20 loss:0.063 grdn:0.484 lr:9.7e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 07:28:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:28:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:32:59 ts/train.py:394 step:3K smpl:435K ep:786 epch:14.03 loss:0.061 grdn:0.430 lr:9.7e-05 updt_s:1.280 data_s:0.002
-WARNING 2025-09-10 07:32:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:32:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:37:21 ts/train.py:394 step:4K smpl:461K ep:832 epch:14.85 loss:0.061 grdn:0.435 lr:9.7e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 07:37:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:37:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:41:44 ts/train.py:394 step:4K smpl:486K ep:878 epch:15.68 loss:0.059 grdn:0.408 lr:9.6e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 07:41:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:41:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:46:08 ts/train.py:394 step:4K smpl:512K ep:924 epch:16.50 loss:0.057 grdn:0.392 lr:9.6e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 07:46:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:46:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:46:08 ts/train.py:403 Checkpoint policy after step 4000
-INFO 2025-09-10 07:50:34 ts/train.py:394 step:4K smpl:538K ep:970 epch:17.33 loss:0.056 grdn:0.384 lr:9.6e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 07:50:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:50:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:54:59 ts/train.py:394 step:4K smpl:563K ep:1K epch:18.15 loss:0.055 grdn:0.356 lr:9.5e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 07:54:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:54:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:59:20 ts/train.py:394 step:5K smpl:589K ep:1K epch:18.98 loss:0.054 grdn:0.361 lr:9.5e-05 updt_s:1.279 data_s:0.002
-WARNING 2025-09-10 07:59:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:59:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:03:42 ts/train.py:394 step:5K smpl:614K ep:1K epch:19.80 loss:0.053 grdn:0.353 lr:9.4e-05 updt_s:1.271 data_s:0.013
-WARNING 2025-09-10 08:03:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:03:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:08:05 ts/train.py:394 step:5K smpl:640K ep:1K epch:20.63 loss:0.052 grdn:0.344 lr:9.4e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 08:08:05 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:08:05 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:08:05 ts/train.py:403 Checkpoint policy after step 5000
-INFO 2025-09-10 08:12:28 ts/train.py:394 step:5K smpl:666K ep:1K epch:21.45 loss:0.051 grdn:0.343 lr:9.3e-05 updt_s:1.268 data_s:0.013
-WARNING 2025-09-10 08:12:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:12:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:16:50 ts/train.py:394 step:5K smpl:691K ep:1K epch:22.28 loss:0.050 grdn:0.316 lr:9.3e-05 updt_s:1.271 data_s:0.014
-WARNING 2025-09-10 08:16:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:16:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:21:15 ts/train.py:394 step:6K smpl:717K ep:1K epch:23.10 loss:0.049 grdn:0.334 lr:9.2e-05 updt_s:1.287 data_s:0.014
-WARNING 2025-09-10 08:21:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:21:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:25:38 ts/train.py:394 step:6K smpl:742K ep:1K epch:23.93 loss:0.048 grdn:0.316 lr:9.2e-05 updt_s:1.289 data_s:0.002
-WARNING 2025-09-10 08:25:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:25:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:30:01 ts/train.py:394 step:6K smpl:768K ep:1K epch:24.75 loss:0.047 grdn:0.307 lr:9.1e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 08:30:01 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:30:01 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:30:01 ts/train.py:403 Checkpoint policy after step 6000
-INFO 2025-09-10 08:34:26 ts/train.py:394 step:6K smpl:794K ep:1K epch:25.58 loss:0.046 grdn:0.303 lr:9.0e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 08:34:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:34:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:38:47 ts/train.py:394 step:6K smpl:819K ep:1K epch:26.40 loss:0.045 grdn:0.290 lr:9.0e-05 updt_s:1.268 data_s:0.014
-WARNING 2025-09-10 08:38:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:38:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:43:10 ts/train.py:394 step:7K smpl:845K ep:2K epch:27.23 loss:0.044 grdn:0.286 lr:8.9e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 08:43:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:43:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:47:30 ts/train.py:394 step:7K smpl:870K ep:2K epch:28.05 loss:0.044 grdn:0.288 lr:8.8e-05 updt_s:1.275 data_s:0.002
-WARNING 2025-09-10 08:47:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:47:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:51:53 ts/train.py:394 step:7K smpl:896K ep:2K epch:28.88 loss:0.043 grdn:0.274 lr:8.8e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 08:51:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:51:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:51:53 ts/train.py:403 Checkpoint policy after step 7000
-INFO 2025-09-10 08:56:19 ts/train.py:394 step:7K smpl:922K ep:2K epch:29.70 loss:0.042 grdn:0.268 lr:8.7e-05 updt_s:1.282 data_s:0.014
-WARNING 2025-09-10 08:56:19 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:56:19 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:00:43 ts/train.py:394 step:7K smpl:947K ep:2K epch:30.53 loss:0.041 grdn:0.271 lr:8.6e-05 updt_s:1.283 data_s:0.014
-WARNING 2025-09-10 09:00:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:00:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:05:07 ts/train.py:394 step:8K smpl:973K ep:2K epch:31.35 loss:0.040 grdn:0.268 lr:8.6e-05 updt_s:1.279 data_s:0.014
-WARNING 2025-09-10 09:05:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:05:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:09:31 ts/train.py:394 step:8K smpl:998K ep:2K epch:32.18 loss:0.040 grdn:0.263 lr:8.5e-05 updt_s:1.282 data_s:0.014
-WARNING 2025-09-10 09:09:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:09:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:13:52 ts/train.py:394 step:8K smpl:1M ep:2K epch:33.00 loss:0.039 grdn:0.277 lr:8.4e-05 updt_s:1.276 data_s:0.002
-WARNING 2025-09-10 09:13:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:13:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:13:52 ts/train.py:403 Checkpoint policy after step 8000
-INFO 2025-09-10 09:18:15 ts/train.py:394 step:8K smpl:1M ep:2K epch:33.83 loss:0.038 grdn:0.258 lr:8.3e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 09:18:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:18:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:22:37 ts/train.py:394 step:8K smpl:1M ep:2K epch:34.65 loss:0.038 grdn:0.255 lr:8.3e-05 updt_s:1.267 data_s:0.014
-WARNING 2025-09-10 09:22:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:22:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:26:58 ts/train.py:394 step:9K smpl:1M ep:2K epch:35.48 loss:0.037 grdn:0.239 lr:8.2e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 09:26:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:26:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:31:20 ts/train.py:394 step:9K smpl:1M ep:2K epch:36.31 loss:0.036 grdn:0.246 lr:8.1e-05 updt_s:1.270 data_s:0.013
-WARNING 2025-09-10 09:31:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:31:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:35:42 ts/train.py:394 step:9K smpl:1M ep:2K epch:37.13 loss:0.036 grdn:0.244 lr:8.0e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 09:35:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:35:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:35:42 ts/train.py:403 Checkpoint policy after step 9000
-INFO 2025-09-10 09:40:03 ts/train.py:394 step:9K smpl:1M ep:2K epch:37.96 loss:0.035 grdn:0.242 lr:7.9e-05 updt_s:1.272 data_s:0.002
-WARNING 2025-09-10 09:40:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:40:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:44:24 ts/train.py:394 step:9K smpl:1M ep:2K epch:38.78 loss:0.034 grdn:0.233 lr:7.9e-05 updt_s:1.268 data_s:0.014
-WARNING 2025-09-10 09:44:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:44:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:48:47 ts/train.py:394 step:10K smpl:1M ep:2K epch:39.61 loss:0.033 grdn:0.232 lr:7.8e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 09:48:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:48:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:53:08 ts/train.py:394 step:10K smpl:1M ep:2K epch:40.43 loss:0.033 grdn:0.226 lr:7.7e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 09:53:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:53:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:57:30 ts/train.py:394 step:10K smpl:1M ep:2K epch:41.26 loss:0.033 grdn:0.220 lr:7.6e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 09:57:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:57:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:57:30 ts/train.py:403 Checkpoint policy after step 10000
-INFO 2025-09-10 10:01:52 ts/train.py:394 step:10K smpl:1M ep:2K epch:42.08 loss:0.032 grdn:0.229 lr:7.5e-05 updt_s:1.276 data_s:0.002
-WARNING 2025-09-10 10:01:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:01:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:06:14 ts/train.py:394 step:10K smpl:1M ep:2K epch:42.91 loss:0.031 grdn:0.219 lr:7.4e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 10:06:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:06:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:10:35 ts/train.py:394 step:11K smpl:1M ep:2K epch:43.73 loss:0.031 grdn:0.214 lr:7.3e-05 updt_s:1.268 data_s:0.014
-WARNING 2025-09-10 10:10:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:10:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:14:58 ts/train.py:394 step:11K smpl:1M ep:2K epch:44.56 loss:0.030 grdn:0.218 lr:7.2e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 10:14:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:14:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:19:20 ts/train.py:394 step:11K smpl:1M ep:3K epch:45.38 loss:0.030 grdn:0.223 lr:7.2e-05 updt_s:1.271 data_s:0.014
-WARNING 2025-09-10 10:19:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:19:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:19:20 ts/train.py:403 Checkpoint policy after step 11000
-INFO 2025-09-10 10:23:47 ts/train.py:394 step:11K smpl:1M ep:3K epch:46.21 loss:0.030 grdn:0.209 lr:7.1e-05 updt_s:1.285 data_s:0.014
-WARNING 2025-09-10 10:23:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:23:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:28:08 ts/train.py:394 step:11K smpl:1M ep:3K epch:47.03 loss:0.029 grdn:0.209 lr:7.0e-05 updt_s:1.279 data_s:0.002
-WARNING 2025-09-10 10:28:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:28:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:32:31 ts/train.py:394 step:12K smpl:1M ep:3K epch:47.86 loss:0.028 grdn:0.205 lr:6.9e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 10:32:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:32:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:36:54 ts/train.py:394 step:12K smpl:2M ep:3K epch:48.68 loss:0.028 grdn:0.203 lr:6.8e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 10:36:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:36:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:41:18 ts/train.py:394 step:12K smpl:2M ep:3K epch:49.51 loss:0.028 grdn:0.209 lr:6.7e-05 updt_s:1.283 data_s:0.014
-WARNING 2025-09-10 10:41:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:41:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:41:18 ts/train.py:403 Checkpoint policy after step 12000
-INFO 2025-09-10 10:45:43 ts/train.py:394 step:12K smpl:2M ep:3K epch:50.33 loss:0.027 grdn:0.199 lr:6.6e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 10:45:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:45:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:50:08 ts/train.py:394 step:12K smpl:2M ep:3K epch:51.16 loss:0.027 grdn:0.201 lr:6.5e-05 updt_s:1.288 data_s:0.013
-WARNING 2025-09-10 10:50:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:50:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:54:32 ts/train.py:394 step:13K smpl:2M ep:3K epch:51.98 loss:0.027 grdn:0.201 lr:6.4e-05 updt_s:1.290 data_s:0.002
-WARNING 2025-09-10 10:54:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:54:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:58:53 ts/train.py:394 step:13K smpl:2M ep:3K epch:52.81 loss:0.026 grdn:0.203 lr:6.3e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 10:58:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:58:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:03:16 ts/train.py:394 step:13K smpl:2M ep:3K epch:53.63 loss:0.026 grdn:0.198 lr:6.2e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 11:03:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:03:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:03:16 ts/train.py:403 Checkpoint policy after step 13000
-INFO 2025-09-10 11:07:40 ts/train.py:394 step:13K smpl:2M ep:3K epch:54.46 loss:0.026 grdn:0.197 lr:6.1e-05 updt_s:1.273 data_s:0.014
-WARNING 2025-09-10 11:07:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:07:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:12:03 ts/train.py:394 step:13K smpl:2M ep:3K epch:55.28 loss:0.026 grdn:0.196 lr:6.0e-05 updt_s:1.277 data_s:0.013
-WARNING 2025-09-10 11:12:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:12:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:16:25 ts/train.py:394 step:14K smpl:2M ep:3K epch:56.11 loss:0.025 grdn:0.188 lr:5.9e-05 updt_s:1.283 data_s:0.002
-WARNING 2025-09-10 11:16:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:16:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:20:48 ts/train.py:394 step:14K smpl:2M ep:3K epch:56.93 loss:0.025 grdn:0.192 lr:5.8e-05 updt_s:1.279 data_s:0.014
-WARNING 2025-09-10 11:20:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:20:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:25:10 ts/train.py:394 step:14K smpl:2M ep:3K epch:57.76 loss:0.025 grdn:0.188 lr:5.7e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 11:25:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:25:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:25:10 ts/train.py:403 Checkpoint policy after step 14000
-INFO 2025-09-10 11:29:35 ts/train.py:394 step:14K smpl:2M ep:3K epch:58.58 loss:0.024 grdn:0.189 lr:5.6e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 11:29:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:29:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:34:00 ts/train.py:394 step:14K smpl:2M ep:3K epch:59.41 loss:0.024 grdn:0.185 lr:5.5e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 11:34:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:34:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:38:23 ts/train.py:394 step:15K smpl:2M ep:3K epch:60.23 loss:0.024 grdn:0.186 lr:5.4e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 11:38:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:38:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:42:45 ts/train.py:394 step:15K smpl:2M ep:3K epch:61.06 loss:0.024 grdn:0.184 lr:5.3e-05 updt_s:1.282 data_s:0.002
-WARNING 2025-09-10 11:42:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:42:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:47:05 ts/train.py:394 step:15K smpl:2M ep:3K epch:61.88 loss:0.023 grdn:0.182 lr:5.2e-05 updt_s:1.263 data_s:0.014
-WARNING 2025-09-10 11:47:05 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:47:05 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:47:05 ts/train.py:403 Checkpoint policy after step 15000
-INFO 2025-09-10 11:51:31 ts/train.py:394 step:15K smpl:2M ep:4K epch:62.71 loss:0.023 grdn:0.184 lr:5.1e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 11:51:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:51:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:55:52 ts/train.py:394 step:15K smpl:2M ep:4K epch:63.53 loss:0.023 grdn:0.183 lr:5.0e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 11:55:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:55:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:00:16 ts/train.py:394 step:16K smpl:2M ep:4K epch:64.36 loss:0.023 grdn:0.179 lr:4.9e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 12:00:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:00:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:04:41 ts/train.py:394 step:16K smpl:2M ep:4K epch:65.18 loss:0.023 grdn:0.179 lr:4.8e-05 updt_s:1.286 data_s:0.014
-WARNING 2025-09-10 12:04:41 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:04:41 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:09:03 ts/train.py:394 step:16K smpl:2M ep:4K epch:66.01 loss:0.022 grdn:0.177 lr:4.7e-05 updt_s:1.282 data_s:0.002
-WARNING 2025-09-10 12:09:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:09:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:09:03 ts/train.py:403 Checkpoint policy after step 16000
-INFO 2025-09-10 12:13:28 ts/train.py:394 step:16K smpl:2M ep:4K epch:66.83 loss:0.022 grdn:0.181 lr:4.6e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 12:13:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:13:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:17:49 ts/train.py:394 step:16K smpl:2M ep:4K epch:67.66 loss:0.022 grdn:0.175 lr:4.5e-05 updt_s:1.265 data_s:0.014
-WARNING 2025-09-10 12:17:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:17:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:22:12 ts/train.py:394 step:17K smpl:2M ep:4K epch:68.48 loss:0.022 grdn:0.174 lr:4.4e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 12:22:12 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:22:12 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:26:37 ts/train.py:394 step:17K smpl:2M ep:4K epch:69.31 loss:0.022 grdn:0.172 lr:4.3e-05 updt_s:1.286 data_s:0.014
-WARNING 2025-09-10 12:26:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:26:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:31:00 ts/train.py:394 step:17K smpl:2M ep:4K epch:70.13 loss:0.021 grdn:0.171 lr:4.2e-05 updt_s:1.285 data_s:0.002
-WARNING 2025-09-10 12:31:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:31:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:31:00 ts/train.py:403 Checkpoint policy after step 17000
-INFO 2025-09-10 12:35:24 ts/train.py:394 step:17K smpl:2M ep:4K epch:70.96 loss:0.022 grdn:0.174 lr:4.1e-05 updt_s:1.273 data_s:0.014
-WARNING 2025-09-10 12:35:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:35:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:39:47 ts/train.py:394 step:17K smpl:2M ep:4K epch:71.78 loss:0.021 grdn:0.167 lr:4.0e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 12:39:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:39:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:44:09 ts/train.py:394 step:18K smpl:2M ep:4K epch:72.61 loss:0.021 grdn:0.168 lr:3.9e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 12:44:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:44:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:48:32 ts/train.py:394 step:18K smpl:2M ep:4K epch:73.44 loss:0.021 grdn:0.171 lr:3.8e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 12:48:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:48:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:52:55 ts/train.py:394 step:18K smpl:2M ep:4K epch:74.26 loss:0.021 grdn:0.166 lr:3.7e-05 updt_s:1.279 data_s:0.014
-WARNING 2025-09-10 12:52:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:52:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:52:55 ts/train.py:403 Checkpoint policy after step 18000
-INFO 2025-09-10 12:57:18 ts/train.py:394 step:18K smpl:2M ep:4K epch:75.09 loss:0.021 grdn:0.166 lr:3.6e-05 updt_s:1.278 data_s:0.002
-WARNING 2025-09-10 12:57:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:57:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:01:43 ts/train.py:394 step:18K smpl:2M ep:4K epch:75.91 loss:0.021 grdn:0.167 lr:3.5e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 13:01:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:01:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:06:04 ts/train.py:394 step:19K smpl:2M ep:4K epch:76.74 loss:0.021 grdn:0.167 lr:3.4e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 13:06:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:06:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:10:26 ts/train.py:394 step:19K smpl:2M ep:4K epch:77.56 loss:0.020 grdn:0.167 lr:3.3e-05 updt_s:1.271 data_s:0.014
-WARNING 2025-09-10 13:10:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:10:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:14:48 ts/train.py:394 step:19K smpl:2M ep:4K epch:78.39 loss:0.020 grdn:0.168 lr:3.2e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 13:14:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:14:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:14:48 ts/train.py:403 Checkpoint policy after step 19000
-INFO 2025-09-10 13:19:12 ts/train.py:394 step:19K smpl:2M ep:4K epch:79.21 loss:0.020 grdn:0.159 lr:3.1e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 13:19:12 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:19:12 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:23:34 ts/train.py:394 step:19K smpl:2M ep:4K epch:80.04 loss:0.020 grdn:0.164 lr:3.0e-05 updt_s:1.281 data_s:0.002
-WARNING 2025-09-10 13:23:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:23:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:27:57 ts/train.py:394 step:20K smpl:3M ep:5K epch:80.86 loss:0.020 grdn:0.162 lr:2.9e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 13:27:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:27:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:32:21 ts/train.py:394 step:20K smpl:3M ep:5K epch:81.69 loss:0.020 grdn:0.165 lr:2.8e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 13:32:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:32:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:36:44 ts/train.py:394 step:20K smpl:3M ep:5K epch:82.51 loss:0.020 grdn:0.162 lr:2.7e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 13:36:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:36:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:36:44 ts/train.py:403 Checkpoint policy after step 20000
-INFO 2025-09-10 13:36:45 ts/train.py:445 End of training

wandb/latest-run/files/requirements.txt

@@ -1,140 +0,0 @@
-orderly-set==5.4.1
-evdev==1.9.2
-sympy==1.14.0
-mergedeep==1.3.4
-platformdirs==4.3.8
-numba==0.61.2
-sentry-sdk==2.32.0
-PySocks==1.7.1
-frozenlist==1.7.0
-nvidia-nccl-cu12==2.26.2
-nvidia-cuda-cupti-cu12==12.6.80
-python-xlib==0.33
-transformers==4.53.1
-protobuf==6.31.1
-certifi==2025.7.9
-pfzy==0.3.4
-torch==2.7.1
-huggingface-hub==0.33.2
-itsdangerous==2.2.0
-exceptiongroup==1.3.0
-beautifulsoup4==4.13.4
-PyYAML==6.0.2
-inquirerpy==0.3.4
-datasets==3.6.0
-filelock==3.18.0
-cffi==1.17.1
-multidict==6.6.3
-pluggy==1.6.0
-fasteners==0.19
-lerobot==0.1.0
-pydantic_core==2.33.2
-av==15.0.0
-aiohappyeyeballs==2.6.1
-Farama-Notifications==0.0.4
-pyarrow==20.0.0
-imageio-ffmpeg==0.6.0
-aiosignal==1.4.0
-pyserial==3.5
-num2words==0.5.14
-draccus==0.10.0
-numpy==2.2.6
-asciitree==0.3.3
-gdown==5.2.0
-pycparser==2.22
-six==1.17.0
-pydantic==2.11.7
-accelerate==1.8.1
-pip==25.1
-multiprocess==0.70.16
-numcodecs==0.13.1
-aiohttp==3.12.14
-nvidia-cublas-cu12==12.6.4.1
-torchcodec==0.4.0
-pandas==2.3.1
-tokenizers==0.21.2
-GitPython==3.1.44
-termcolor==3.1.0
-typing_extensions==4.14.1
-click==8.2.1
-deepdiff==8.5.0
-diffusers==0.34.0
-hf-xet==1.1.5
-typing-inspect==0.9.0
-wcwidth==0.2.13
-Werkzeug==3.1.3
-iniconfig==2.1.0
-Pygments==2.19.2
-antlr4-python3-runtime==4.9.3
-gymnasium==0.29.1
-blinker==1.9.0
-cloudpickle==3.1.1
-fsspec==2025.3.0
-yarl==1.20.1
-setuptools==78.1.1
-tzdata==2025.2
-safetensors==0.5.3
-nvidia-cusparse-cu12==12.5.4.2
-pyyaml-include==1.4.1
-soupsieve==2.7
-idna==3.10
-nvidia-cudnn-cu12==9.5.1.17
-pyzmq==27.0.0
-llvmlite==0.44.0
-jsonlines==4.0.0
-MarkupSafe==3.0.2
-pymunk==6.11.1
-pillow==11.3.0
-einops==0.8.1
-nvidia-cusolver-cu12==11.7.1.2
-nvidia-cuda-nvrtc-cu12==12.6.77
-torchvision==0.22.1
-wheel==0.45.1
-wandb==0.21.0
-zipp==3.23.0
-nvidia-cufile-cu12==1.11.1.6
-imageio==2.37.0
-opencv-python-headless==4.12.0.88
-toml==0.10.2
-hf_transfer==0.1.9
-nvidia-curand-cu12==10.3.7.77
-gitdb==4.0.12
-importlib_metadata==8.7.0
-nvidia-cufft-cu12==11.3.0.4
-zarr==2.18.3
-prompt_toolkit==3.0.51
-psutil==7.0.0
-smmap==5.0.2
-packaging==25.0
-xxhash==3.5.0
-attrs==25.3.0
-nvidia-nvjitlink-cu12==12.6.85
-pytest==8.4.1
-nvidia-cuda-runtime-cu12==12.6.77
-omegaconf==2.3.0
-annotated-types==0.7.0
-pytz==2025.2
-mypy_extensions==1.1.0
-networkx==3.4.2
-h5py==3.14.0
-Flask==3.1.1
-nvidia-cusparselt-cu12==0.6.3
-tomli==2.2.1
-nvidia-nvtx-cu12==12.6.77
-docopt==0.6.2
-urllib3==2.5.0
-async-timeout==5.0.1
-rerun-sdk==0.23.4
-cmake==4.0.3
-regex==2024.11.6
-python-dateutil==2.9.0.post0
-Jinja2==3.1.6
-pynput==1.8.1
-charset-normalizer==3.4.2
-requests==2.32.4
-typing-inspection==0.4.1
-triton==3.3.1
-tqdm==4.67.1
-propcache==0.3.2
-dill==0.3.8
-mpmath==1.3.0

wandb/latest-run/files/wandb-metadata.json

@@ -1,46 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1031-aws-x86_64-with-glibc2.39",
-  "python":  "CPython 3.10.13",
-  "startedAt":  "2025-09-10T06:17:53.891577Z",
-  "args":  [
-    "--policy.path=lerobot/smolvla_base",
-    "--config_path=src/training_cfg/insert_rope/svla_rel.json",
-    "--dataset.repo_id=black_table_insert_rope_downsampled_v7",
-    "--job_name=2025-09-10_06-17-38_svla_rel_v7",
-    "--output_dir=outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7"
-  ],
-  "program":  "/home/ubuntu/projects/policy/lerobot/scripts/train.py",
-  "codePath":  "lerobot/scripts/train.py",
-  "codePathLocal":  "lerobot/scripts/train.py",
-  "git":  {
-    "remote":  "git@github.com:SceniX-AI/policy.git",
-    "commit":  "cc0d6bb844d05c377b053051ed40114478c3a7c8"
-  },
-  "root":  "outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7",
-  "host":  "ip-172-31-8-128",
-  "executable":  "/home/ubuntu/miniconda3/envs/lerobot/bin/python",
-  "cpu_count":  4,
-  "cpu_count_logical":  8,
-  "gpu":  "NVIDIA L40S",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "1063956480000",
-      "used":  "581626609664"
-    }
-  },
-  "memory":  {
-    "total":  "66507816960"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA L40S",
-      "memoryTotal":  "48305799168",
-      "cudaCores":  18176,
-      "architecture":  "Ada",
-      "uuid":  "GPU-9277cabf-a4d3-fab1-cd41-888c1e1f5b3a"
-    }
-  ],
-  "cudaVersion":  "12.8",
-  "writerId":  "k282cdjuxylro9nzqrdyy1qm8gwgxyoa"
-}

wandb/latest-run/files/wandb-summary.json

@@ -1 +0,0 @@
-{"train/lr":2.7316655657931005e-05,"train/epochs":82.51144201637337,"_step":20000,"train/samples":2560000,"train/dataloading_s":0.01383356748996448,"train/grad_norm":0.1624235799908638,"train/episodes":4620.640752916908,"train/loss":0.01839064247906208,"_runtime":26331.912255248,"train/update_s":1.2762478608399397,"train/steps":20000,"_timestamp":1.757511404013634e+09,"_wandb":{"runtime":26331}}

wandb/latest-run/logs/debug-core.log

@@ -1,14 +0,0 @@
-{"time":"2025-09-10T06:17:54.247298159Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpst5j99vf/port-2168.txt","pid":2168,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
-{"time":"2025-09-10T06:17:54.249963816Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2168}
-{"time":"2025-09-10T06:17:54.25070645Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2168-2225-3406642978/socket","Net":"unix"}}
-{"time":"2025-09-10T06:17:54.308029503Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
-{"time":"2025-09-10T06:17:54.318124952Z","level":"INFO","msg":"handleInformInit: received","streamId":"od103fqv","id":"1(@)"}
-{"time":"2025-09-10T06:17:54.493588505Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"od103fqv","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.54536493Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.545435121Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.545457262Z","level":"INFO","msg":"server is shutting down"}
-{"time":"2025-09-10T13:36:46.545492952Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.545562644Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2168-2225-3406642978/socket","Net":"unix"}}
-{"time":"2025-09-10T13:36:46.887525917Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.887547337Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.887554597Z","level":"INFO","msg":"server is closed"}

wandb/latest-run/logs/debug-internal.log

@@ -1,12 +0,0 @@
-{"time":"2025-09-10T06:17:54.318252724Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
-{"time":"2025-09-10T06:17:54.493560254Z","level":"INFO","msg":"stream: created new stream","id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493583685Z","level":"INFO","msg":"stream: started","id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493606835Z","level":"INFO","msg":"writer: Do: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493615595Z","level":"INFO","msg":"handler: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493633765Z","level":"INFO","msg":"sender: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.545468692Z","level":"INFO","msg":"stream: closing","id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.78249679Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-09-10T13:36:46.884073306Z","level":"INFO","msg":"handler: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884106596Z","level":"INFO","msg":"sender: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884095846Z","level":"INFO","msg":"writer: Close: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884184498Z","level":"INFO","msg":"stream: closed","id":"od103fqv"}

wandb/latest-run/logs/debug.log

@@ -1,22 +0,0 @@
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Configure stats pid to 2168
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/projects/policy/wandb/settings
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7/wandb/run-20250910_061753-od103fqv/logs/debug.log
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7/wandb/run-20250910_061753-od103fqv/logs/debug-internal.log
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:init():830] calling init triggers
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
-config: {'dataset': {'repo_id': 'black_table_insert_rope_downsampled_v7', 'root': None, 'episodes': None, 'image_transforms': {'enable': True, 'max_num_transforms': 3, 'random_order': True, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'perspective': {'weight': 1.0, 'type': 'RandomPerspective', 'kwargs': {'distortion_scale': 0.025, 'p': 1.0}}, 'rotation': {'weight': 1.0, 'type': 'RandomRotation', 'kwargs': {'degrees': [-5, 5]}}, 'crop': {'weight': 1.0, 'type': 'RandomCrop', 'kwargs': {'size': [110, 202]}}}, 'downsample': {'resize': {'weight': 1.0, 'type': 'Resize', 'kwargs': {'size': [120, 212]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'relative_actions': True, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [6]}, 'observation.image2': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}, 'observation.image': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}, 'observation.image3': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'optimizer_grad_clip_norm': 10.0, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': True, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': 0, 'pad_language_to': 'max_length', 'num_expert_layers': 0, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0}, 'output_dir': 'outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7', 'job_name': '2025-09-10_06-17-38_svla_rel_v7', 'resume': False, 'seed': 100000, 'num_workers': 4, 'batch_size': 128, 'steps': 20000, 'eval_freq': 2000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 1000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-10, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 0.0001, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
-2025-09-10 06:17:53,893 INFO    MainThread:2168 [wandb_init.py:init():871] starting backend
-2025-09-10 06:17:54,308 INFO    MainThread:2168 [wandb_init.py:init():874] sending inform_init request
-2025-09-10 06:17:54,312 INFO    MainThread:2168 [wandb_init.py:init():882] backend started and connected
-2025-09-10 06:17:54,314 INFO    MainThread:2168 [wandb_init.py:init():953] updated telemetry
-2025-09-10 06:17:54,318 INFO    MainThread:2168 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
-2025-09-10 06:17:54,631 INFO    MainThread:2168 [wandb_init.py:init():1029] starting run threads in backend
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_console_start():2458] atexit reg
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2306] redirect: wrap_raw
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2375] Wrapping output streams.
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2398] Redirects installed.
-2025-09-10 06:17:54,799 INFO    MainThread:2168 [wandb_init.py:init():1075] run started, returning control to user process
-2025-09-10 13:36:46,544 INFO    MsgRouterThr:2168 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

wandb/latest-run/run-od103fqv.wandb

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:521b58dd5e29557cee40ffdc2106b1a0390863d967ba066cf78f15ce551e2804
-size 1715589

wandb/run-20250910_061753-od103fqv/files/config.yaml

@@ -1,280 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.21.0
-        e:
-            k282cdjuxylro9nzqrdyy1qm8gwgxyoa:
-                args:
-                    - --policy.path=lerobot/smolvla_base
-                    - --config_path=src/training_cfg/insert_rope/svla_rel.json
-                    - --dataset.repo_id=black_table_insert_rope_downsampled_v7
-                    - --job_name=2025-09-10_06-17-38_svla_rel_v7
-                    - --output_dir=outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-                codePath: lerobot/scripts/train.py
-                codePathLocal: lerobot/scripts/train.py
-                cpu_count: 4
-                cpu_count_logical: 8
-                cudaVersion: "12.8"
-                disk:
-                    /:
-                        total: "1063956480000"
-                        used: "581626609664"
-                executable: /home/ubuntu/miniconda3/envs/lerobot/bin/python
-                git:
-                    commit: cc0d6bb844d05c377b053051ed40114478c3a7c8
-                    remote: git@github.com:SceniX-AI/policy.git
-                gpu: NVIDIA L40S
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Ada
-                      cudaCores: 18176
-                      memoryTotal: "48305799168"
-                      name: NVIDIA L40S
-                      uuid: GPU-9277cabf-a4d3-fab1-cd41-888c1e1f5b3a
-                host: ip-172-31-8-128
-                memory:
-                    total: "66507816960"
-                os: Linux-6.8.0-1031-aws-x86_64-with-glibc2.39
-                program: /home/ubuntu/projects/policy/lerobot/scripts/train.py
-                python: CPython 3.10.13
-                root: outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-                startedAt: "2025-09-10T06:17:53.891577Z"
-                writerId: k282cdjuxylro9nzqrdyy1qm8gwgxyoa
-        m: []
-        python_version: 3.10.13
-        t:
-            "1":
-                - 1
-                - 11
-                - 41
-                - 49
-                - 51
-                - 71
-            "2":
-                - 1
-                - 11
-                - 41
-                - 49
-                - 51
-                - 71
-            "3":
-                - 13
-                - 15
-                - 16
-                - 61
-            "4": 3.10.13
-            "5": 0.21.0
-            "6": 4.53.1
-            "10":
-                - 21
-            "12": 0.21.0
-            "13": linux-x86_64
-batch_size:
-    value: 128
-dataset:
-    value:
-        episodes: null
-        image_transforms:
-            downsample:
-                resize:
-                    kwargs:
-                        size:
-                            - 120
-                            - 212
-                    type: Resize
-                    weight: 1
-            enable: true
-            max_num_transforms: 3
-            random_order: true
-            tfs:
-                brightness:
-                    kwargs:
-                        brightness:
-                            - 0.8
-                            - 1.2
-                    type: ColorJitter
-                    weight: 1
-                contrast:
-                    kwargs:
-                        contrast:
-                            - 0.8
-                            - 1.2
-                    type: ColorJitter
-                    weight: 1
-                crop:
-                    kwargs:
-                        size:
-                            - 110
-                            - 202
-                    type: RandomCrop
-                    weight: 1
-                hue:
-                    kwargs:
-                        hue:
-                            - -0.05
-                            - 0.05
-                    type: ColorJitter
-                    weight: 1
-                perspective:
-                    kwargs:
-                        distortion_scale: 0.025
-                        p: 1
-                    type: RandomPerspective
-                    weight: 1
-                rotation:
-                    kwargs:
-                        degrees:
-                            - -5
-                            - 5
-                    type: RandomRotation
-                    weight: 1
-                saturation:
-                    kwargs:
-                        saturation:
-                            - 0.5
-                            - 1.5
-                    type: ColorJitter
-                    weight: 1
-                sharpness:
-                    kwargs:
-                        sharpness:
-                            - 0.5
-                            - 1.5
-                    type: SharpnessJitter
-                    weight: 1
-        repo_id: black_table_insert_rope_downsampled_v7
-        revision: null
-        root: null
-        use_imagenet_stats: true
-        video_backend: torchcodec
-env:
-    value: null
-eval:
-    value:
-        batch_size: 50
-        n_episodes: 50
-        use_async_envs: false
-eval_freq:
-    value: 2000
-job_name:
-    value: 2025-09-10_06-17-38_svla_rel_v7
-log_freq:
-    value: 200
-num_workers:
-    value: 4
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        eps: 1e-08
-        grad_clip_norm: 10
-        lr: 0.0001
-        type: adamw
-        weight_decay: 1e-10
-output_dir:
-    value: outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-policy:
-    value:
-        adapt_to_pi_aloha: false
-        add_image_special_tokens: false
-        attention_mode: cross_attn
-        chunk_size: 50
-        device: cuda
-        empty_cameras: 0
-        expert_width_multiplier: 0.75
-        freeze_vision_encoder: true
-        input_features:
-            observation.image:
-                shape:
-                    - 3
-                    - 256
-                    - 256
-                type: VISUAL
-            observation.image2:
-                shape:
-                    - 3
-                    - 256
-                    - 256
-                type: VISUAL
-            observation.image3:
-                shape:
-                    - 3
-                    - 256
-                    - 256
-                type: VISUAL
-            observation.state:
-                shape:
-                    - 6
-                type: STATE
-        load_vlm_weights: true
-        max_action_dim: 32
-        max_period: 4
-        max_state_dim: 32
-        min_period: 0.004
-        n_action_steps: 50
-        n_obs_steps: 1
-        normalization_mapping:
-            ACTION: MEAN_STD
-            STATE: MEAN_STD
-            VISUAL: IDENTITY
-        num_expert_layers: 0
-        num_steps: 10
-        num_vlm_layers: 16
-        optimizer_betas:
-            - 0.9
-            - 0.95
-        optimizer_eps: 1e-08
-        optimizer_grad_clip_norm: 10
-        optimizer_lr: 0.0001
-        optimizer_weight_decay: 1e-10
-        output_features:
-            action:
-                shape:
-                    - 6
-                type: ACTION
-        pad_language_to: max_length
-        prefix_length: 0
-        relative_actions: true
-        resize_imgs_with_padding:
-            - 512
-            - 512
-        scheduler_decay_lr: 2.5e-06
-        scheduler_decay_steps: 30000
-        scheduler_warmup_steps: 1000
-        self_attn_every_n_layers: 2
-        tokenizer_max_length: 48
-        train_expert_only: true
-        train_state_proj: true
-        type: smolvla
-        use_amp: false
-        use_cache: true
-        use_delta_joint_actions_aloha: false
-        vlm_model_name: HuggingFaceTB/SmolVLM2-500M-Video-Instruct
-resume:
-    value: false
-save_checkpoint:
-    value: true
-save_freq:
-    value: 1000
-scheduler:
-    value:
-        decay_lr: 2.5e-06
-        num_decay_steps: 30000
-        num_warmup_steps: 1000
-        peak_lr: 0.0001
-        type: cosine_decay_with_warmup
-seed:
-    value: 100000
-steps:
-    value: 20000
-use_policy_training_preset:
-    value: true
-wandb:
-    value:
-        disable_artifact: true
-        enable: true
-        entity: null
-        mode: null
-        notes: null
-        project: lerobot
-        run_id: null

wandb/run-20250910_061753-od103fqv/files/output.log

@@ -1,341 +0,0 @@
-Logs will be synced with wandb.
-INFO 2025-09-10 06:17:54 ndb_utils.py:96 Track this run --> https://wandb.ai/ss7050-columbia/lerobot/runs/od103fqv
-INFO 2025-09-10 06:17:54 ts/train.py:260 Creating dataset
-Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 56/56 [00:00<00:00, 291054.55it/s]
-Downloading data: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 56/56 [00:00<00:00, 317406.79files/s]
-Generating train split: 31026 examples [00:00, 368550.14 examples/s]
-INFO 2025-09-10 06:17:55 ts/train.py:272 Creating policy
-INFO 2025-09-10 06:17:55 ts/train.py:275 Using relative actions: True
-You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0.
-Loading  HuggingFaceTB/SmolVLM2-500M-Video-Instruct weights ...
-INFO 2025-09-10 06:17:57 modeling.py:991 We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).
-Reducing the number of VLM layers to 16 ...
-INFO 2025-09-10 06:18:23 ts/train.py:303 Creating optimizer and scheduler
-INFO 2025-09-10 06:18:23 ts/train.py:315 Output dir: outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7
-INFO 2025-09-10 06:18:23 ts/train.py:318 cfg.steps=20000 (20K)
-INFO 2025-09-10 06:18:23 ts/train.py:319 dataset.num_frames=31026 (31K)
-INFO 2025-09-10 06:18:23 ts/train.py:320 dataset.num_episodes=56
-INFO 2025-09-10 06:18:23 ts/train.py:321 num_learnable_params=99880992 (100M)
-INFO 2025-09-10 06:18:23 ts/train.py:322 num_total_params=450047792 (450M)
-INFO 2025-09-10 06:18:23 ts/train.py:361 Start offline training on a fixed dataset
-INFO 2025-09-10 06:22:48 ts/train.py:394 step:200 smpl:26K ep:46 epch:0.83 loss:1.131 grdn:2.910 lr:1.0e-05 updt_s:1.287 data_s:0.015
-WARNING 2025-09-10 06:22:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:22:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:27:13 ts/train.py:394 step:400 smpl:51K ep:92 epch:1.65 loss:0.316 grdn:2.630 lr:3.0e-05 updt_s:1.288 data_s:0.013
-WARNING 2025-09-10 06:27:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:27:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:31:35 ts/train.py:394 step:600 smpl:77K ep:139 epch:2.48 loss:0.164 grdn:3.080 lr:5.0e-05 updt_s:1.273 data_s:0.013
-WARNING 2025-09-10 06:31:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:31:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:35:58 ts/train.py:394 step:800 smpl:102K ep:185 epch:3.30 loss:0.135 grdn:3.063 lr:7.0e-05 updt_s:1.278 data_s:0.013
-WARNING 2025-09-10 06:35:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:35:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:40:20 ts/train.py:394 step:1K smpl:128K ep:231 epch:4.13 loss:0.119 grdn:2.490 lr:9.0e-05 updt_s:1.267 data_s:0.014
-WARNING 2025-09-10 06:40:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:40:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:40:20 ts/train.py:403 Checkpoint policy after step 1000
-INFO 2025-09-10 06:44:43 ts/train.py:394 step:1K smpl:154K ep:277 epch:4.95 loss:0.106 grdn:1.772 lr:1.0e-04 updt_s:1.280 data_s:0.002
-WARNING 2025-09-10 06:44:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:44:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:49:06 ts/train.py:394 step:1K smpl:179K ep:323 epch:5.78 loss:0.096 grdn:1.347 lr:1.0e-04 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 06:49:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:49:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:53:31 ts/train.py:394 step:2K smpl:205K ep:370 epch:6.60 loss:0.088 grdn:1.062 lr:9.9e-05 updt_s:1.287 data_s:0.013
-WARNING 2025-09-10 06:53:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:53:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 06:57:54 ts/train.py:394 step:2K smpl:230K ep:416 epch:7.43 loss:0.084 grdn:0.930 lr:9.9e-05 updt_s:1.278 data_s:0.013
-WARNING 2025-09-10 06:57:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 06:57:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:02:18 ts/train.py:394 step:2K smpl:256K ep:462 epch:8.25 loss:0.079 grdn:0.797 lr:9.9e-05 updt_s:1.282 data_s:0.014
-WARNING 2025-09-10 07:02:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:02:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:02:18 ts/train.py:403 Checkpoint policy after step 2000
-INFO 2025-09-10 07:06:45 ts/train.py:394 step:2K smpl:282K ep:508 epch:9.08 loss:0.075 grdn:0.692 lr:9.9e-05 updt_s:1.286 data_s:0.014
-WARNING 2025-09-10 07:06:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:06:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:11:06 ts/train.py:394 step:2K smpl:307K ep:554 epch:9.90 loss:0.072 grdn:0.628 lr:9.9e-05 updt_s:1.280 data_s:0.002
-WARNING 2025-09-10 07:11:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:11:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:15:26 ts/train.py:394 step:3K smpl:333K ep:601 epch:10.73 loss:0.069 grdn:0.593 lr:9.8e-05 updt_s:1.262 data_s:0.014
-WARNING 2025-09-10 07:15:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:15:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:19:49 ts/train.py:394 step:3K smpl:358K ep:647 epch:11.55 loss:0.067 grdn:0.531 lr:9.8e-05 updt_s:1.276 data_s:0.013
-WARNING 2025-09-10 07:19:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:19:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:24:13 ts/train.py:394 step:3K smpl:384K ep:693 epch:12.38 loss:0.065 grdn:0.509 lr:9.8e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 07:24:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:24:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:24:13 ts/train.py:403 Checkpoint policy after step 3000
-INFO 2025-09-10 07:28:38 ts/train.py:394 step:3K smpl:410K ep:739 epch:13.20 loss:0.063 grdn:0.484 lr:9.7e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 07:28:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:28:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:32:59 ts/train.py:394 step:3K smpl:435K ep:786 epch:14.03 loss:0.061 grdn:0.430 lr:9.7e-05 updt_s:1.280 data_s:0.002
-WARNING 2025-09-10 07:32:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:32:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:37:21 ts/train.py:394 step:4K smpl:461K ep:832 epch:14.85 loss:0.061 grdn:0.435 lr:9.7e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 07:37:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:37:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:41:44 ts/train.py:394 step:4K smpl:486K ep:878 epch:15.68 loss:0.059 grdn:0.408 lr:9.6e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 07:41:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:41:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:46:08 ts/train.py:394 step:4K smpl:512K ep:924 epch:16.50 loss:0.057 grdn:0.392 lr:9.6e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 07:46:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:46:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:46:08 ts/train.py:403 Checkpoint policy after step 4000
-INFO 2025-09-10 07:50:34 ts/train.py:394 step:4K smpl:538K ep:970 epch:17.33 loss:0.056 grdn:0.384 lr:9.6e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 07:50:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:50:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:54:59 ts/train.py:394 step:4K smpl:563K ep:1K epch:18.15 loss:0.055 grdn:0.356 lr:9.5e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 07:54:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:54:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 07:59:20 ts/train.py:394 step:5K smpl:589K ep:1K epch:18.98 loss:0.054 grdn:0.361 lr:9.5e-05 updt_s:1.279 data_s:0.002
-WARNING 2025-09-10 07:59:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 07:59:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:03:42 ts/train.py:394 step:5K smpl:614K ep:1K epch:19.80 loss:0.053 grdn:0.353 lr:9.4e-05 updt_s:1.271 data_s:0.013
-WARNING 2025-09-10 08:03:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:03:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:08:05 ts/train.py:394 step:5K smpl:640K ep:1K epch:20.63 loss:0.052 grdn:0.344 lr:9.4e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 08:08:05 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:08:05 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:08:05 ts/train.py:403 Checkpoint policy after step 5000
-INFO 2025-09-10 08:12:28 ts/train.py:394 step:5K smpl:666K ep:1K epch:21.45 loss:0.051 grdn:0.343 lr:9.3e-05 updt_s:1.268 data_s:0.013
-WARNING 2025-09-10 08:12:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:12:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:16:50 ts/train.py:394 step:5K smpl:691K ep:1K epch:22.28 loss:0.050 grdn:0.316 lr:9.3e-05 updt_s:1.271 data_s:0.014
-WARNING 2025-09-10 08:16:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:16:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:21:15 ts/train.py:394 step:6K smpl:717K ep:1K epch:23.10 loss:0.049 grdn:0.334 lr:9.2e-05 updt_s:1.287 data_s:0.014
-WARNING 2025-09-10 08:21:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:21:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:25:38 ts/train.py:394 step:6K smpl:742K ep:1K epch:23.93 loss:0.048 grdn:0.316 lr:9.2e-05 updt_s:1.289 data_s:0.002
-WARNING 2025-09-10 08:25:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:25:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:30:01 ts/train.py:394 step:6K smpl:768K ep:1K epch:24.75 loss:0.047 grdn:0.307 lr:9.1e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 08:30:01 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:30:01 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:30:01 ts/train.py:403 Checkpoint policy after step 6000
-INFO 2025-09-10 08:34:26 ts/train.py:394 step:6K smpl:794K ep:1K epch:25.58 loss:0.046 grdn:0.303 lr:9.0e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 08:34:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:34:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:38:47 ts/train.py:394 step:6K smpl:819K ep:1K epch:26.40 loss:0.045 grdn:0.290 lr:9.0e-05 updt_s:1.268 data_s:0.014
-WARNING 2025-09-10 08:38:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:38:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:43:10 ts/train.py:394 step:7K smpl:845K ep:2K epch:27.23 loss:0.044 grdn:0.286 lr:8.9e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 08:43:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:43:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:47:30 ts/train.py:394 step:7K smpl:870K ep:2K epch:28.05 loss:0.044 grdn:0.288 lr:8.8e-05 updt_s:1.275 data_s:0.002
-WARNING 2025-09-10 08:47:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:47:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:51:53 ts/train.py:394 step:7K smpl:896K ep:2K epch:28.88 loss:0.043 grdn:0.274 lr:8.8e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 08:51:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:51:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 08:51:53 ts/train.py:403 Checkpoint policy after step 7000
-INFO 2025-09-10 08:56:19 ts/train.py:394 step:7K smpl:922K ep:2K epch:29.70 loss:0.042 grdn:0.268 lr:8.7e-05 updt_s:1.282 data_s:0.014
-WARNING 2025-09-10 08:56:19 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 08:56:19 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:00:43 ts/train.py:394 step:7K smpl:947K ep:2K epch:30.53 loss:0.041 grdn:0.271 lr:8.6e-05 updt_s:1.283 data_s:0.014
-WARNING 2025-09-10 09:00:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:00:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:05:07 ts/train.py:394 step:8K smpl:973K ep:2K epch:31.35 loss:0.040 grdn:0.268 lr:8.6e-05 updt_s:1.279 data_s:0.014
-WARNING 2025-09-10 09:05:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:05:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:09:31 ts/train.py:394 step:8K smpl:998K ep:2K epch:32.18 loss:0.040 grdn:0.263 lr:8.5e-05 updt_s:1.282 data_s:0.014
-WARNING 2025-09-10 09:09:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:09:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:13:52 ts/train.py:394 step:8K smpl:1M ep:2K epch:33.00 loss:0.039 grdn:0.277 lr:8.4e-05 updt_s:1.276 data_s:0.002
-WARNING 2025-09-10 09:13:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:13:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:13:52 ts/train.py:403 Checkpoint policy after step 8000
-INFO 2025-09-10 09:18:15 ts/train.py:394 step:8K smpl:1M ep:2K epch:33.83 loss:0.038 grdn:0.258 lr:8.3e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 09:18:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:18:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:22:37 ts/train.py:394 step:8K smpl:1M ep:2K epch:34.65 loss:0.038 grdn:0.255 lr:8.3e-05 updt_s:1.267 data_s:0.014
-WARNING 2025-09-10 09:22:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:22:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:26:58 ts/train.py:394 step:9K smpl:1M ep:2K epch:35.48 loss:0.037 grdn:0.239 lr:8.2e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 09:26:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:26:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:31:20 ts/train.py:394 step:9K smpl:1M ep:2K epch:36.31 loss:0.036 grdn:0.246 lr:8.1e-05 updt_s:1.270 data_s:0.013
-WARNING 2025-09-10 09:31:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:31:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:35:42 ts/train.py:394 step:9K smpl:1M ep:2K epch:37.13 loss:0.036 grdn:0.244 lr:8.0e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 09:35:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:35:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:35:42 ts/train.py:403 Checkpoint policy after step 9000
-INFO 2025-09-10 09:40:03 ts/train.py:394 step:9K smpl:1M ep:2K epch:37.96 loss:0.035 grdn:0.242 lr:7.9e-05 updt_s:1.272 data_s:0.002
-WARNING 2025-09-10 09:40:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:40:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:44:24 ts/train.py:394 step:9K smpl:1M ep:2K epch:38.78 loss:0.034 grdn:0.233 lr:7.9e-05 updt_s:1.268 data_s:0.014
-WARNING 2025-09-10 09:44:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:44:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:48:47 ts/train.py:394 step:10K smpl:1M ep:2K epch:39.61 loss:0.033 grdn:0.232 lr:7.8e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 09:48:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:48:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:53:08 ts/train.py:394 step:10K smpl:1M ep:2K epch:40.43 loss:0.033 grdn:0.226 lr:7.7e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 09:53:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:53:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:57:30 ts/train.py:394 step:10K smpl:1M ep:2K epch:41.26 loss:0.033 grdn:0.220 lr:7.6e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 09:57:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 09:57:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 09:57:30 ts/train.py:403 Checkpoint policy after step 10000
-INFO 2025-09-10 10:01:52 ts/train.py:394 step:10K smpl:1M ep:2K epch:42.08 loss:0.032 grdn:0.229 lr:7.5e-05 updt_s:1.276 data_s:0.002
-WARNING 2025-09-10 10:01:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:01:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:06:14 ts/train.py:394 step:10K smpl:1M ep:2K epch:42.91 loss:0.031 grdn:0.219 lr:7.4e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 10:06:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:06:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:10:35 ts/train.py:394 step:11K smpl:1M ep:2K epch:43.73 loss:0.031 grdn:0.214 lr:7.3e-05 updt_s:1.268 data_s:0.014
-WARNING 2025-09-10 10:10:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:10:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:14:58 ts/train.py:394 step:11K smpl:1M ep:2K epch:44.56 loss:0.030 grdn:0.218 lr:7.2e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 10:14:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:14:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:19:20 ts/train.py:394 step:11K smpl:1M ep:3K epch:45.38 loss:0.030 grdn:0.223 lr:7.2e-05 updt_s:1.271 data_s:0.014
-WARNING 2025-09-10 10:19:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:19:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:19:20 ts/train.py:403 Checkpoint policy after step 11000
-INFO 2025-09-10 10:23:47 ts/train.py:394 step:11K smpl:1M ep:3K epch:46.21 loss:0.030 grdn:0.209 lr:7.1e-05 updt_s:1.285 data_s:0.014
-WARNING 2025-09-10 10:23:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:23:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:28:08 ts/train.py:394 step:11K smpl:1M ep:3K epch:47.03 loss:0.029 grdn:0.209 lr:7.0e-05 updt_s:1.279 data_s:0.002
-WARNING 2025-09-10 10:28:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:28:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:32:31 ts/train.py:394 step:12K smpl:1M ep:3K epch:47.86 loss:0.028 grdn:0.205 lr:6.9e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 10:32:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:32:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:36:54 ts/train.py:394 step:12K smpl:2M ep:3K epch:48.68 loss:0.028 grdn:0.203 lr:6.8e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 10:36:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:36:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:41:18 ts/train.py:394 step:12K smpl:2M ep:3K epch:49.51 loss:0.028 grdn:0.209 lr:6.7e-05 updt_s:1.283 data_s:0.014
-WARNING 2025-09-10 10:41:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:41:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:41:18 ts/train.py:403 Checkpoint policy after step 12000
-INFO 2025-09-10 10:45:43 ts/train.py:394 step:12K smpl:2M ep:3K epch:50.33 loss:0.027 grdn:0.199 lr:6.6e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 10:45:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:45:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:50:08 ts/train.py:394 step:12K smpl:2M ep:3K epch:51.16 loss:0.027 grdn:0.201 lr:6.5e-05 updt_s:1.288 data_s:0.013
-WARNING 2025-09-10 10:50:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:50:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:54:32 ts/train.py:394 step:13K smpl:2M ep:3K epch:51.98 loss:0.027 grdn:0.201 lr:6.4e-05 updt_s:1.290 data_s:0.002
-WARNING 2025-09-10 10:54:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:54:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 10:58:53 ts/train.py:394 step:13K smpl:2M ep:3K epch:52.81 loss:0.026 grdn:0.203 lr:6.3e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 10:58:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 10:58:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:03:16 ts/train.py:394 step:13K smpl:2M ep:3K epch:53.63 loss:0.026 grdn:0.198 lr:6.2e-05 updt_s:1.272 data_s:0.014
-WARNING 2025-09-10 11:03:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:03:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:03:16 ts/train.py:403 Checkpoint policy after step 13000
-INFO 2025-09-10 11:07:40 ts/train.py:394 step:13K smpl:2M ep:3K epch:54.46 loss:0.026 grdn:0.197 lr:6.1e-05 updt_s:1.273 data_s:0.014
-WARNING 2025-09-10 11:07:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:07:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:12:03 ts/train.py:394 step:13K smpl:2M ep:3K epch:55.28 loss:0.026 grdn:0.196 lr:6.0e-05 updt_s:1.277 data_s:0.013
-WARNING 2025-09-10 11:12:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:12:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:16:25 ts/train.py:394 step:14K smpl:2M ep:3K epch:56.11 loss:0.025 grdn:0.188 lr:5.9e-05 updt_s:1.283 data_s:0.002
-WARNING 2025-09-10 11:16:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:16:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:20:48 ts/train.py:394 step:14K smpl:2M ep:3K epch:56.93 loss:0.025 grdn:0.192 lr:5.8e-05 updt_s:1.279 data_s:0.014
-WARNING 2025-09-10 11:20:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:20:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:25:10 ts/train.py:394 step:14K smpl:2M ep:3K epch:57.76 loss:0.025 grdn:0.188 lr:5.7e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 11:25:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:25:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:25:10 ts/train.py:403 Checkpoint policy after step 14000
-INFO 2025-09-10 11:29:35 ts/train.py:394 step:14K smpl:2M ep:3K epch:58.58 loss:0.024 grdn:0.189 lr:5.6e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 11:29:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:29:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:34:00 ts/train.py:394 step:14K smpl:2M ep:3K epch:59.41 loss:0.024 grdn:0.185 lr:5.5e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 11:34:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:34:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:38:23 ts/train.py:394 step:15K smpl:2M ep:3K epch:60.23 loss:0.024 grdn:0.186 lr:5.4e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 11:38:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:38:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:42:45 ts/train.py:394 step:15K smpl:2M ep:3K epch:61.06 loss:0.024 grdn:0.184 lr:5.3e-05 updt_s:1.282 data_s:0.002
-WARNING 2025-09-10 11:42:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:42:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:47:05 ts/train.py:394 step:15K smpl:2M ep:3K epch:61.88 loss:0.023 grdn:0.182 lr:5.2e-05 updt_s:1.263 data_s:0.014
-WARNING 2025-09-10 11:47:05 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:47:05 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:47:05 ts/train.py:403 Checkpoint policy after step 15000
-INFO 2025-09-10 11:51:31 ts/train.py:394 step:15K smpl:2M ep:4K epch:62.71 loss:0.023 grdn:0.184 lr:5.1e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 11:51:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:51:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 11:55:52 ts/train.py:394 step:15K smpl:2M ep:4K epch:63.53 loss:0.023 grdn:0.183 lr:5.0e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 11:55:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 11:55:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:00:16 ts/train.py:394 step:16K smpl:2M ep:4K epch:64.36 loss:0.023 grdn:0.179 lr:4.9e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 12:00:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:00:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:04:41 ts/train.py:394 step:16K smpl:2M ep:4K epch:65.18 loss:0.023 grdn:0.179 lr:4.8e-05 updt_s:1.286 data_s:0.014
-WARNING 2025-09-10 12:04:41 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:04:41 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:09:03 ts/train.py:394 step:16K smpl:2M ep:4K epch:66.01 loss:0.022 grdn:0.177 lr:4.7e-05 updt_s:1.282 data_s:0.002
-WARNING 2025-09-10 12:09:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:09:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:09:03 ts/train.py:403 Checkpoint policy after step 16000
-INFO 2025-09-10 12:13:28 ts/train.py:394 step:16K smpl:2M ep:4K epch:66.83 loss:0.022 grdn:0.181 lr:4.6e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 12:13:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:13:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:17:49 ts/train.py:394 step:16K smpl:2M ep:4K epch:67.66 loss:0.022 grdn:0.175 lr:4.5e-05 updt_s:1.265 data_s:0.014
-WARNING 2025-09-10 12:17:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:17:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:22:12 ts/train.py:394 step:17K smpl:2M ep:4K epch:68.48 loss:0.022 grdn:0.174 lr:4.4e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 12:22:12 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:22:12 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:26:37 ts/train.py:394 step:17K smpl:2M ep:4K epch:69.31 loss:0.022 grdn:0.172 lr:4.3e-05 updt_s:1.286 data_s:0.014
-WARNING 2025-09-10 12:26:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:26:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:31:00 ts/train.py:394 step:17K smpl:2M ep:4K epch:70.13 loss:0.021 grdn:0.171 lr:4.2e-05 updt_s:1.285 data_s:0.002
-WARNING 2025-09-10 12:31:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:31:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:31:00 ts/train.py:403 Checkpoint policy after step 17000
-INFO 2025-09-10 12:35:24 ts/train.py:394 step:17K smpl:2M ep:4K epch:70.96 loss:0.022 grdn:0.174 lr:4.1e-05 updt_s:1.273 data_s:0.014
-WARNING 2025-09-10 12:35:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:35:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:39:47 ts/train.py:394 step:17K smpl:2M ep:4K epch:71.78 loss:0.021 grdn:0.167 lr:4.0e-05 updt_s:1.278 data_s:0.014
-WARNING 2025-09-10 12:39:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:39:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:44:09 ts/train.py:394 step:18K smpl:2M ep:4K epch:72.61 loss:0.021 grdn:0.168 lr:3.9e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 12:44:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:44:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:48:32 ts/train.py:394 step:18K smpl:2M ep:4K epch:73.44 loss:0.021 grdn:0.171 lr:3.8e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 12:48:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:48:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:52:55 ts/train.py:394 step:18K smpl:2M ep:4K epch:74.26 loss:0.021 grdn:0.166 lr:3.7e-05 updt_s:1.279 data_s:0.014
-WARNING 2025-09-10 12:52:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:52:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 12:52:55 ts/train.py:403 Checkpoint policy after step 18000
-INFO 2025-09-10 12:57:18 ts/train.py:394 step:18K smpl:2M ep:4K epch:75.09 loss:0.021 grdn:0.166 lr:3.6e-05 updt_s:1.278 data_s:0.002
-WARNING 2025-09-10 12:57:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 12:57:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:01:43 ts/train.py:394 step:18K smpl:2M ep:4K epch:75.91 loss:0.021 grdn:0.167 lr:3.5e-05 updt_s:1.284 data_s:0.014
-WARNING 2025-09-10 13:01:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:01:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:06:04 ts/train.py:394 step:19K smpl:2M ep:4K epch:76.74 loss:0.021 grdn:0.167 lr:3.4e-05 updt_s:1.269 data_s:0.014
-WARNING 2025-09-10 13:06:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:06:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:10:26 ts/train.py:394 step:19K smpl:2M ep:4K epch:77.56 loss:0.020 grdn:0.167 lr:3.3e-05 updt_s:1.271 data_s:0.014
-WARNING 2025-09-10 13:10:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:10:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:14:48 ts/train.py:394 step:19K smpl:2M ep:4K epch:78.39 loss:0.020 grdn:0.168 lr:3.2e-05 updt_s:1.270 data_s:0.014
-WARNING 2025-09-10 13:14:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:14:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:14:48 ts/train.py:403 Checkpoint policy after step 19000
-INFO 2025-09-10 13:19:12 ts/train.py:394 step:19K smpl:2M ep:4K epch:79.21 loss:0.020 grdn:0.159 lr:3.1e-05 updt_s:1.275 data_s:0.014
-WARNING 2025-09-10 13:19:12 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:19:12 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:23:34 ts/train.py:394 step:19K smpl:2M ep:4K epch:80.04 loss:0.020 grdn:0.164 lr:3.0e-05 updt_s:1.281 data_s:0.002
-WARNING 2025-09-10 13:23:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:23:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:27:57 ts/train.py:394 step:20K smpl:3M ep:5K epch:80.86 loss:0.020 grdn:0.162 lr:2.9e-05 updt_s:1.277 data_s:0.014
-WARNING 2025-09-10 13:27:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:27:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:32:21 ts/train.py:394 step:20K smpl:3M ep:5K epch:81.69 loss:0.020 grdn:0.165 lr:2.8e-05 updt_s:1.280 data_s:0.014
-WARNING 2025-09-10 13:32:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:32:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:36:44 ts/train.py:394 step:20K smpl:3M ep:5K epch:82.51 loss:0.020 grdn:0.162 lr:2.7e-05 updt_s:1.276 data_s:0.014
-WARNING 2025-09-10 13:36:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-WARNING 2025-09-10 13:36:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
-INFO 2025-09-10 13:36:44 ts/train.py:403 Checkpoint policy after step 20000
-INFO 2025-09-10 13:36:45 ts/train.py:445 End of training

wandb/run-20250910_061753-od103fqv/files/requirements.txt

@@ -1,140 +0,0 @@
-orderly-set==5.4.1
-evdev==1.9.2
-sympy==1.14.0
-mergedeep==1.3.4
-platformdirs==4.3.8
-numba==0.61.2
-sentry-sdk==2.32.0
-PySocks==1.7.1
-frozenlist==1.7.0
-nvidia-nccl-cu12==2.26.2
-nvidia-cuda-cupti-cu12==12.6.80
-python-xlib==0.33
-transformers==4.53.1
-protobuf==6.31.1
-certifi==2025.7.9
-pfzy==0.3.4
-torch==2.7.1
-huggingface-hub==0.33.2
-itsdangerous==2.2.0
-exceptiongroup==1.3.0
-beautifulsoup4==4.13.4
-PyYAML==6.0.2
-inquirerpy==0.3.4
-datasets==3.6.0
-filelock==3.18.0
-cffi==1.17.1
-multidict==6.6.3
-pluggy==1.6.0
-fasteners==0.19
-lerobot==0.1.0
-pydantic_core==2.33.2
-av==15.0.0
-aiohappyeyeballs==2.6.1
-Farama-Notifications==0.0.4
-pyarrow==20.0.0
-imageio-ffmpeg==0.6.0
-aiosignal==1.4.0
-pyserial==3.5
-num2words==0.5.14
-draccus==0.10.0
-numpy==2.2.6
-asciitree==0.3.3
-gdown==5.2.0
-pycparser==2.22
-six==1.17.0
-pydantic==2.11.7
-accelerate==1.8.1
-pip==25.1
-multiprocess==0.70.16
-numcodecs==0.13.1
-aiohttp==3.12.14
-nvidia-cublas-cu12==12.6.4.1
-torchcodec==0.4.0
-pandas==2.3.1
-tokenizers==0.21.2
-GitPython==3.1.44
-termcolor==3.1.0
-typing_extensions==4.14.1
-click==8.2.1
-deepdiff==8.5.0
-diffusers==0.34.0
-hf-xet==1.1.5
-typing-inspect==0.9.0
-wcwidth==0.2.13
-Werkzeug==3.1.3
-iniconfig==2.1.0
-Pygments==2.19.2
-antlr4-python3-runtime==4.9.3
-gymnasium==0.29.1
-blinker==1.9.0
-cloudpickle==3.1.1
-fsspec==2025.3.0
-yarl==1.20.1
-setuptools==78.1.1
-tzdata==2025.2
-safetensors==0.5.3
-nvidia-cusparse-cu12==12.5.4.2
-pyyaml-include==1.4.1
-soupsieve==2.7
-idna==3.10
-nvidia-cudnn-cu12==9.5.1.17
-pyzmq==27.0.0
-llvmlite==0.44.0
-jsonlines==4.0.0
-MarkupSafe==3.0.2
-pymunk==6.11.1
-pillow==11.3.0
-einops==0.8.1
-nvidia-cusolver-cu12==11.7.1.2
-nvidia-cuda-nvrtc-cu12==12.6.77
-torchvision==0.22.1
-wheel==0.45.1
-wandb==0.21.0
-zipp==3.23.0
-nvidia-cufile-cu12==1.11.1.6
-imageio==2.37.0
-opencv-python-headless==4.12.0.88
-toml==0.10.2
-hf_transfer==0.1.9
-nvidia-curand-cu12==10.3.7.77
-gitdb==4.0.12
-importlib_metadata==8.7.0
-nvidia-cufft-cu12==11.3.0.4
-zarr==2.18.3
-prompt_toolkit==3.0.51
-psutil==7.0.0
-smmap==5.0.2
-packaging==25.0
-xxhash==3.5.0
-attrs==25.3.0
-nvidia-nvjitlink-cu12==12.6.85
-pytest==8.4.1
-nvidia-cuda-runtime-cu12==12.6.77
-omegaconf==2.3.0
-annotated-types==0.7.0
-pytz==2025.2
-mypy_extensions==1.1.0
-networkx==3.4.2
-h5py==3.14.0
-Flask==3.1.1
-nvidia-cusparselt-cu12==0.6.3
-tomli==2.2.1
-nvidia-nvtx-cu12==12.6.77
-docopt==0.6.2
-urllib3==2.5.0
-async-timeout==5.0.1
-rerun-sdk==0.23.4
-cmake==4.0.3
-regex==2024.11.6
-python-dateutil==2.9.0.post0
-Jinja2==3.1.6
-pynput==1.8.1
-charset-normalizer==3.4.2
-requests==2.32.4
-typing-inspection==0.4.1
-triton==3.3.1
-tqdm==4.67.1
-propcache==0.3.2
-dill==0.3.8
-mpmath==1.3.0

wandb/run-20250910_061753-od103fqv/files/wandb-metadata.json

@@ -1,46 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1031-aws-x86_64-with-glibc2.39",
-  "python":  "CPython 3.10.13",
-  "startedAt":  "2025-09-10T06:17:53.891577Z",
-  "args":  [
-    "--policy.path=lerobot/smolvla_base",
-    "--config_path=src/training_cfg/insert_rope/svla_rel.json",
-    "--dataset.repo_id=black_table_insert_rope_downsampled_v7",
-    "--job_name=2025-09-10_06-17-38_svla_rel_v7",
-    "--output_dir=outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7"
-  ],
-  "program":  "/home/ubuntu/projects/policy/lerobot/scripts/train.py",
-  "codePath":  "lerobot/scripts/train.py",
-  "codePathLocal":  "lerobot/scripts/train.py",
-  "git":  {
-    "remote":  "git@github.com:SceniX-AI/policy.git",
-    "commit":  "cc0d6bb844d05c377b053051ed40114478c3a7c8"
-  },
-  "root":  "outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7",
-  "host":  "ip-172-31-8-128",
-  "executable":  "/home/ubuntu/miniconda3/envs/lerobot/bin/python",
-  "cpu_count":  4,
-  "cpu_count_logical":  8,
-  "gpu":  "NVIDIA L40S",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "1063956480000",
-      "used":  "581626609664"
-    }
-  },
-  "memory":  {
-    "total":  "66507816960"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA L40S",
-      "memoryTotal":  "48305799168",
-      "cudaCores":  18176,
-      "architecture":  "Ada",
-      "uuid":  "GPU-9277cabf-a4d3-fab1-cd41-888c1e1f5b3a"
-    }
-  ],
-  "cudaVersion":  "12.8",
-  "writerId":  "k282cdjuxylro9nzqrdyy1qm8gwgxyoa"
-}

wandb/run-20250910_061753-od103fqv/files/wandb-summary.json

@@ -1 +0,0 @@
-{"train/lr":2.7316655657931005e-05,"train/epochs":82.51144201637337,"_step":20000,"train/samples":2560000,"train/dataloading_s":0.01383356748996448,"train/grad_norm":0.1624235799908638,"train/episodes":4620.640752916908,"train/loss":0.01839064247906208,"_runtime":26331.912255248,"train/update_s":1.2762478608399397,"train/steps":20000,"_timestamp":1.757511404013634e+09,"_wandb":{"runtime":26331}}

wandb/run-20250910_061753-od103fqv/logs/debug-core.log

@@ -1,14 +0,0 @@
-{"time":"2025-09-10T06:17:54.247298159Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpst5j99vf/port-2168.txt","pid":2168,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
-{"time":"2025-09-10T06:17:54.249963816Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2168}
-{"time":"2025-09-10T06:17:54.25070645Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2168-2225-3406642978/socket","Net":"unix"}}
-{"time":"2025-09-10T06:17:54.308029503Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
-{"time":"2025-09-10T06:17:54.318124952Z","level":"INFO","msg":"handleInformInit: received","streamId":"od103fqv","id":"1(@)"}
-{"time":"2025-09-10T06:17:54.493588505Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"od103fqv","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.54536493Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.545435121Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.545457262Z","level":"INFO","msg":"server is shutting down"}
-{"time":"2025-09-10T13:36:46.545492952Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.545562644Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2168-2225-3406642978/socket","Net":"unix"}}
-{"time":"2025-09-10T13:36:46.887525917Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.887547337Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
-{"time":"2025-09-10T13:36:46.887554597Z","level":"INFO","msg":"server is closed"}

wandb/run-20250910_061753-od103fqv/logs/debug-internal.log

@@ -1,12 +0,0 @@
-{"time":"2025-09-10T06:17:54.318252724Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
-{"time":"2025-09-10T06:17:54.493560254Z","level":"INFO","msg":"stream: created new stream","id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493583685Z","level":"INFO","msg":"stream: started","id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493606835Z","level":"INFO","msg":"writer: Do: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493615595Z","level":"INFO","msg":"handler: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T06:17:54.493633765Z","level":"INFO","msg":"sender: started","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.545468692Z","level":"INFO","msg":"stream: closing","id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.78249679Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-09-10T13:36:46.884073306Z","level":"INFO","msg":"handler: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884106596Z","level":"INFO","msg":"sender: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884095846Z","level":"INFO","msg":"writer: Close: closed","stream_id":"od103fqv"}
-{"time":"2025-09-10T13:36:46.884184498Z","level":"INFO","msg":"stream: closed","id":"od103fqv"}

wandb/run-20250910_061753-od103fqv/logs/debug.log

@@ -1,22 +0,0 @@
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Configure stats pid to 2168
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/projects/policy/wandb/settings
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7/wandb/run-20250910_061753-od103fqv/logs/debug.log
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7/wandb/run-20250910_061753-od103fqv/logs/debug-internal.log
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:init():830] calling init triggers
-2025-09-10 06:17:53,892 INFO    MainThread:2168 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
-config: {'dataset': {'repo_id': 'black_table_insert_rope_downsampled_v7', 'root': None, 'episodes': None, 'image_transforms': {'enable': True, 'max_num_transforms': 3, 'random_order': True, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'perspective': {'weight': 1.0, 'type': 'RandomPerspective', 'kwargs': {'distortion_scale': 0.025, 'p': 1.0}}, 'rotation': {'weight': 1.0, 'type': 'RandomRotation', 'kwargs': {'degrees': [-5, 5]}}, 'crop': {'weight': 1.0, 'type': 'RandomCrop', 'kwargs': {'size': [110, 202]}}}, 'downsample': {'resize': {'weight': 1.0, 'type': 'Resize', 'kwargs': {'size': [120, 212]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'relative_actions': True, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [6]}, 'observation.image2': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}, 'observation.image': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}, 'observation.image3': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 256, 256]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'optimizer_grad_clip_norm': 10.0, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': True, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': 0, 'pad_language_to': 'max_length', 'num_expert_layers': 0, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0}, 'output_dir': 'outputs/train/black_table_insert_rope_downsampled_v7/2025-09-10_06-17-38_svla_rel_v7', 'job_name': '2025-09-10_06-17-38_svla_rel_v7', 'resume': False, 'seed': 100000, 'num_workers': 4, 'batch_size': 128, 'steps': 20000, 'eval_freq': 2000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 1000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-10, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 0.0001, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
-2025-09-10 06:17:53,893 INFO    MainThread:2168 [wandb_init.py:init():871] starting backend
-2025-09-10 06:17:54,308 INFO    MainThread:2168 [wandb_init.py:init():874] sending inform_init request
-2025-09-10 06:17:54,312 INFO    MainThread:2168 [wandb_init.py:init():882] backend started and connected
-2025-09-10 06:17:54,314 INFO    MainThread:2168 [wandb_init.py:init():953] updated telemetry
-2025-09-10 06:17:54,318 INFO    MainThread:2168 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
-2025-09-10 06:17:54,631 INFO    MainThread:2168 [wandb_init.py:init():1029] starting run threads in backend
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_console_start():2458] atexit reg
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2306] redirect: wrap_raw
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2375] Wrapping output streams.
-2025-09-10 06:17:54,795 INFO    MainThread:2168 [wandb_run.py:_redirect():2398] Redirects installed.
-2025-09-10 06:17:54,799 INFO    MainThread:2168 [wandb_init.py:init():1075] run started, returning control to user process
-2025-09-10 13:36:46,544 INFO    MsgRouterThr:2168 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

wandb/run-20250910_061753-od103fqv/run-od103fqv.wandb

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:521b58dd5e29557cee40ffdc2106b1a0390863d967ba066cf78f15ce551e2804
-size 1715589