Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+smolvla_omy/wandb/run-20251106_162149-8gs9of7r/run-8gs9of7r.wandb filter=lfs diff=lfs merge=lfs -text
+smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/run-p7vw7943.wandb filter=lfs diff=lfs merge=lfs -text

smolvla_omy/checkpoints/005000/pretrained_model/config.json

@@ -0,0 +1,83 @@
+{
+    "type": "smolvla",
+    "n_obs_steps": 1,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "MEAN_STD",
+        "ACTION": "MEAN_STD"
+    },
+    "input_features": {
+        "observation.image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.wrist_image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "chunk_size": 5,
+    "n_action_steps": 5,
+    "max_state_dim": 32,
+    "max_action_dim": 32,
+    "resize_imgs_with_padding": [
+        512,
+        512
+    ],
+    "empty_cameras": 0,
+    "adapt_to_pi_aloha": false,
+    "use_delta_joint_actions_aloha": false,
+    "tokenizer_max_length": 48,
+    "num_steps": 10,
+    "use_cache": true,
+    "freeze_vision_encoder": true,
+    "train_expert_only": true,
+    "train_state_proj": true,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.95
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-10,
+    "optimizer_grad_clip_norm": 10,
+    "scheduler_warmup_steps": 1000,
+    "scheduler_decay_steps": 30000,
+    "scheduler_decay_lr": 2.5e-06,
+    "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+    "load_vlm_weights": false,
+    "add_image_special_tokens": false,
+    "attention_mode": "cross_attn",
+    "prefix_length": -1,
+    "pad_language_to": "longest",
+    "num_expert_layers": -1,
+    "num_vlm_layers": 16,
+    "self_attn_every_n_layers": 2,
+    "expert_width_multiplier": 0.75,
+    "min_period": 0.004,
+    "max_period": 4.0
+}

smolvla_omy/checkpoints/005000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:794c736c8a4faf3263611cc5cf8327f2fe3772773c68b23757596bcfb2d99580
+size 1197790032

smolvla_omy/checkpoints/005000/pretrained_model/train_config.json

@@ -0,0 +1,195 @@
+{
+    "dataset": {
+        "repo_id": "omy_pnp_language",
+        "root": "./demo_data_language",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "smolvla",
+        "n_obs_steps": 1,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "MEAN_STD",
+            "ACTION": "MEAN_STD"
+        },
+        "input_features": {
+            "observation.image": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.wrist_image": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "chunk_size": 5,
+        "n_action_steps": 5,
+        "max_state_dim": 32,
+        "max_action_dim": 32,
+        "resize_imgs_with_padding": [
+            512,
+            512
+        ],
+        "empty_cameras": 0,
+        "adapt_to_pi_aloha": false,
+        "use_delta_joint_actions_aloha": false,
+        "tokenizer_max_length": 48,
+        "num_steps": 10,
+        "use_cache": true,
+        "freeze_vision_encoder": true,
+        "train_expert_only": true,
+        "train_state_proj": true,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.95
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-10,
+        "optimizer_grad_clip_norm": 10,
+        "scheduler_warmup_steps": 1000,
+        "scheduler_decay_steps": 30000,
+        "scheduler_decay_lr": 2.5e-06,
+        "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+        "load_vlm_weights": false,
+        "add_image_special_tokens": false,
+        "attention_mode": "cross_attn",
+        "prefix_length": -1,
+        "pad_language_to": "longest",
+        "num_expert_layers": -1,
+        "num_vlm_layers": 16,
+        "self_attn_every_n_layers": 2,
+        "expert_width_multiplier": 0.75,
+        "min_period": 0.004,
+        "max_period": 4.0
+    },
+    "output_dir": "ckpt/smolvla_omy",
+    "job_name": "smolvla_omy",
+    "resume": false,
+    "seed": 42,
+    "num_workers": 8,
+    "batch_size": 16,
+    "steps": 20000,
+    "eval_freq": -1,
+    "log_freq": 50,
+    "save_checkpoint": true,
+    "save_freq": 5000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-10,
+        "grad_clip_norm": 10,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "cosine_decay_with_warmup",
+        "num_warmup_steps": 1000,
+        "num_decay_steps": 30000,
+        "peak_lr": 0.0001,
+        "decay_lr": 2.5e-06
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "smolvla_omy",
+        "entity": "ai-liber",
+        "notes": null,
+        "run_id": null,
+        "mode": null
+    }
+}

smolvla_omy/checkpoints/005000/training_state/optimizer_param_groups.json

@@ -0,0 +1,526 @@
+[
+    {
+        "lr": 9.34687384344914e-05,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-10,
+        "amsgrad": false,
+        "foreach": null,
+        "maximize": false,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310,
+            311,
+            312,
+            313,
+            314,
+            315,
+            316,
+            317,
+            318,
+            319,
+            320,
+            321,
+            322,
+            323,
+            324,
+            325,
+            326,
+            327,
+            328,
+            329,
+            330,
+            331,
+            332,
+            333,
+            334,
+            335,
+            336,
+            337,
+            338,
+            339,
+            340,
+            341,
+            342,
+            343,
+            344,
+            345,
+            346,
+            347,
+            348,
+            349,
+            350,
+            351,
+            352,
+            353,
+            354,
+            355,
+            356,
+            357,
+            358,
+            359,
+            360,
+            361,
+            362,
+            363,
+            364,
+            365,
+            366,
+            367,
+            368,
+            369,
+            370,
+            371,
+            372,
+            373,
+            374,
+            375,
+            376,
+            377,
+            378,
+            379,
+            380,
+            381,
+            382,
+            383,
+            384,
+            385,
+            386,
+            387,
+            388,
+            389,
+            390,
+            391,
+            392,
+            393,
+            394,
+            395,
+            396,
+            397,
+            398,
+            399,
+            400,
+            401,
+            402,
+            403,
+            404,
+            405,
+            406,
+            407,
+            408,
+            409,
+            410,
+            411,
+            412,
+            413,
+            414,
+            415,
+            416,
+            417,
+            418,
+            419,
+            420,
+            421,
+            422,
+            423,
+            424,
+            425,
+            426,
+            427,
+            428,
+            429,
+            430,
+            431,
+            432,
+            433,
+            434,
+            435,
+            436,
+            437,
+            438,
+            439,
+            440,
+            441,
+            442,
+            443,
+            444,
+            445,
+            446,
+            447,
+            448,
+            449,
+            450,
+            451,
+            452,
+            453,
+            454,
+            455,
+            456,
+            457,
+            458,
+            459,
+            460,
+            461,
+            462,
+            463,
+            464,
+            465,
+            466,
+            467,
+            468,
+            469,
+            470,
+            471,
+            472,
+            473,
+            474,
+            475,
+            476,
+            477,
+            478,
+            479,
+            480,
+            481,
+            482,
+            483,
+            484,
+            485,
+            486,
+            487,
+            488,
+            489,
+            490,
+            491,
+            492,
+            493,
+            494,
+            495,
+            496,
+            497,
+            498,
+            499,
+            500,
+            501,
+            502,
+            503,
+            504,
+            505
+        ]
+    }
+]

smolvla_omy/checkpoints/005000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28dbf586dce9a55eb5a3987438be5f06488031966430c8e6279de94992d23109
+size 412659164

smolvla_omy/checkpoints/005000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe5806563981ea1faee236e400cf9c5eca89fef62fed6aa1a580b085a4cb11e
+size 15708

smolvla_omy/checkpoints/005000/training_state/scheduler_state.json

@@ -0,0 +1,15 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 5000,
+    "verbose": false,
+    "_step_count": 5001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        9.34687384344914e-05
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

smolvla_omy/checkpoints/005000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 5000
+}

smolvla_omy/wandb/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2025-11-06T16:21:49.810658746+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2025-11-06T16:21:50.381496059+08:00","level":"INFO","msg":"stream: created new stream","id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.381670461+08:00","level":"INFO","msg":"handler: started","stream_id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.38195187+08:00","level":"INFO","msg":"stream: started","id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.381970995+08:00","level":"INFO","msg":"writer: started","stream_id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.381975054+08:00","level":"INFO","msg":"sender: started","stream_id":"8gs9of7r"}
+{"time":"2025-11-06T18:28:51.825582909+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-11-06T19:15:51.940995974+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}

smolvla_omy/wandb/debug.log

@@ -0,0 +1,21 @@
+2025-11-06 16:21:49,585 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Configure stats pid to 528220
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Loading settings from /workspace/liber/embodied_ai/lerobot-mujoco-tutorial/wandb/settings
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:setup_run_log_directory():706] Logging user logs to ckpt/smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug.log
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to ckpt/smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug-internal.log
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:init():833] calling init triggers
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'omy_pnp_language', 'root': './demo_data_language', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 5, 'n_action_steps': 5, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'optimizer_grad_clip_norm': 10, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': False, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': -1, 'pad_language_to': 'longest', 'num_expert_layers': -1, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0}, 'output_dir': 'ckpt/smolvla_omy', 'job_name': 'smolvla_omy', 'resume': False, 'seed': 42, 'num_workers': 8, 'batch_size': 16, 'steps': 20000, 'eval_freq': -1, 'log_freq': 50, 'save_checkpoint': True, 'save_freq': 5000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-10, 'grad_clip_norm': 10, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 0.0001, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'smolvla_omy', 'entity': 'ai-liber', 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:init():881] starting backend
+2025-11-06 16:21:49,796 INFO    MainThread:528220 [wandb_init.py:init():884] sending inform_init request
+2025-11-06 16:21:49,803 INFO    MainThread:528220 [wandb_init.py:init():892] backend started and connected
+2025-11-06 16:21:49,806 INFO    MainThread:528220 [wandb_init.py:init():962] updated telemetry
+2025-11-06 16:21:49,819 INFO    MainThread:528220 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2025-11-06 16:21:51,055 INFO    MainThread:528220 [wandb_init.py:init():1033] starting run threads in backend
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_console_start():2506] atexit reg
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_redirect():2446] Redirects installed.
+2025-11-06 16:21:51,416 INFO    MainThread:528220 [wandb_init.py:init():1073] run started, returning control to user process

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/files/output.log

@@ -0,0 +1,631 @@
+Logs will be synced with wandb.
+INFO 2025-11-06 16:21:51 ndb_utils.py:96 Track this run --> https://wandb.ai/ai-liber/smolvla_omy/runs/8gs9of7r
+INFO 2025-11-06 16:21:51 in_model.py:127 Creating dataset
+Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20/20 [00:00<00:00, 18149.30it/s]
+INFO 2025-11-06 16:21:52 in_model.py:138 Creating policy
+Reducing the number of VLM layers to 16 ...
+INFO 2025-11-06 16:23:01 in_model.py:148 Creating optimizer and scheduler
+INFO 2025-11-06 16:23:01 in_model.py:160 Output dir: ckpt/smolvla_omy
+INFO 2025-11-06 16:23:01 in_model.py:163 cfg.steps=20000 (20K)
+INFO 2025-11-06 16:23:01 in_model.py:164 dataset.num_frames=2621 (3K)
+INFO 2025-11-06 16:23:01 in_model.py:165 dataset.num_episodes=20
+INFO 2025-11-06 16:23:01 in_model.py:166 num_learnable_params=99880992 (100M)
+INFO 2025-11-06 16:23:01 in_model.py:167 num_total_params=450046216 (450M)
+INFO 2025-11-06 16:23:01 in_model.py:206 Start offline training on a fixed dataset
+INFO 2025-11-06 16:23:52 in_model.py:236 step:50 smpl:800 ep:6 epch:0.31 loss:0.306 grdn:2.807 lr:2.6e-06 updt_s:0.998 data_s:0.022
+WARNING 2025-11-06 16:23:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:23:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:24:41 in_model.py:236 step:100 smpl:2K ep:12 epch:0.61 loss:0.223 grdn:1.959 lr:7.6e-06 updt_s:0.982 data_s:0.000
+WARNING 2025-11-06 16:24:41 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:24:41 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:25:32 in_model.py:236 step:150 smpl:2K ep:18 epch:0.92 loss:0.166 grdn:1.444 lr:1.3e-05 updt_s:1.013 data_s:0.000
+WARNING 2025-11-06 16:25:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:25:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:26:26 in_model.py:236 step:200 smpl:3K ep:24 epch:1.22 loss:0.112 grdn:1.042 lr:1.8e-05 updt_s:1.049 data_s:0.021
+WARNING 2025-11-06 16:26:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:26:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:27:18 in_model.py:236 step:250 smpl:4K ep:31 epch:1.53 loss:0.091 grdn:0.947 lr:2.3e-05 updt_s:1.048 data_s:0.000
+WARNING 2025-11-06 16:27:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:27:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:28:11 in_model.py:236 step:300 smpl:5K ep:37 epch:1.83 loss:0.079 grdn:0.972 lr:2.8e-05 updt_s:1.060 data_s:0.000
+WARNING 2025-11-06 16:28:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:28:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:29:05 in_model.py:236 step:350 smpl:6K ep:43 epch:2.14 loss:0.068 grdn:0.944 lr:3.3e-05 updt_s:1.056 data_s:0.021
+WARNING 2025-11-06 16:29:05 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:29:05 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:29:59 in_model.py:236 step:400 smpl:6K ep:49 epch:2.44 loss:0.062 grdn:0.921 lr:3.8e-05 updt_s:1.063 data_s:0.000
+WARNING 2025-11-06 16:29:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:29:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:30:52 in_model.py:236 step:450 smpl:7K ep:55 epch:2.75 loss:0.062 grdn:0.979 lr:4.3e-05 updt_s:1.062 data_s:0.000
+WARNING 2025-11-06 16:30:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:30:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:31:46 in_model.py:236 step:500 smpl:8K ep:61 epch:3.05 loss:0.055 grdn:0.987 lr:4.8e-05 updt_s:1.055 data_s:0.021
+WARNING 2025-11-06 16:31:46 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:31:46 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:32:39 in_model.py:236 step:550 smpl:9K ep:67 epch:3.36 loss:0.046 grdn:0.843 lr:5.3e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 16:32:39 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:32:39 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:33:32 in_model.py:236 step:600 smpl:10K ep:73 epch:3.66 loss:0.050 grdn:0.955 lr:5.8e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 16:33:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:33:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:34:25 in_model.py:236 step:650 smpl:10K ep:79 epch:3.97 loss:0.050 grdn:1.030 lr:6.3e-05 updt_s:1.058 data_s:0.000
+WARNING 2025-11-06 16:34:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:34:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:35:19 in_model.py:236 step:700 smpl:11K ep:85 epch:4.27 loss:0.044 grdn:0.878 lr:6.8e-05 updt_s:1.055 data_s:0.021
+WARNING 2025-11-06 16:35:19 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:35:19 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:36:12 in_model.py:236 step:750 smpl:12K ep:92 epch:4.58 loss:0.048 grdn:0.911 lr:7.3e-05 updt_s:1.064 data_s:0.000
+WARNING 2025-11-06 16:36:12 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:36:12 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:37:05 in_model.py:236 step:800 smpl:13K ep:98 epch:4.88 loss:0.050 grdn:0.946 lr:7.8e-05 updt_s:1.063 data_s:0.000
+WARNING 2025-11-06 16:37:05 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:37:05 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:37:59 in_model.py:236 step:850 smpl:14K ep:104 epch:5.19 loss:0.045 grdn:0.904 lr:8.3e-05 updt_s:1.056 data_s:0.024
+WARNING 2025-11-06 16:37:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:37:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:38:53 in_model.py:236 step:900 smpl:14K ep:110 epch:5.49 loss:0.056 grdn:1.043 lr:8.8e-05 updt_s:1.062 data_s:0.000
+WARNING 2025-11-06 16:38:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:38:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:39:46 in_model.py:236 step:950 smpl:15K ep:116 epch:5.80 loss:0.056 grdn:1.051 lr:9.3e-05 updt_s:1.058 data_s:0.000
+WARNING 2025-11-06 16:39:46 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:39:46 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:40:40 in_model.py:236 step:1K smpl:16K ep:122 epch:6.10 loss:0.051 grdn:0.937 lr:9.8e-05 updt_s:1.056 data_s:0.020
+WARNING 2025-11-06 16:40:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:40:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:41:33 in_model.py:236 step:1K smpl:17K ep:128 epch:6.41 loss:0.047 grdn:0.883 lr:1.0e-04 updt_s:1.060 data_s:0.000
+WARNING 2025-11-06 16:41:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:41:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:42:26 in_model.py:236 step:1K smpl:18K ep:134 epch:6.71 loss:0.056 grdn:0.951 lr:1.0e-04 updt_s:1.059 data_s:0.000
+WARNING 2025-11-06 16:42:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:42:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:43:20 in_model.py:236 step:1K smpl:18K ep:140 epch:7.02 loss:0.055 grdn:0.955 lr:1.0e-04 updt_s:1.057 data_s:0.022
+WARNING 2025-11-06 16:43:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:43:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:44:13 in_model.py:236 step:1K smpl:19K ep:147 epch:7.33 loss:0.049 grdn:0.953 lr:1.0e-04 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 16:44:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:44:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:45:06 in_model.py:236 step:1K smpl:20K ep:153 epch:7.63 loss:0.045 grdn:0.868 lr:1.0e-04 updt_s:1.062 data_s:0.000
+WARNING 2025-11-06 16:45:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:45:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:45:59 in_model.py:236 step:1K smpl:21K ep:159 epch:7.94 loss:0.053 grdn:0.922 lr:1.0e-04 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 16:45:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:45:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:46:53 in_model.py:236 step:1K smpl:22K ep:165 epch:8.24 loss:0.054 grdn:0.953 lr:1.0e-04 updt_s:1.055 data_s:0.021
+WARNING 2025-11-06 16:46:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:46:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:47:46 in_model.py:236 step:1K smpl:22K ep:171 epch:8.55 loss:0.044 grdn:0.778 lr:9.9e-05 updt_s:1.057 data_s:0.000
+WARNING 2025-11-06 16:47:46 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:47:46 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:48:39 in_model.py:236 step:1K smpl:23K ep:177 epch:8.85 loss:0.044 grdn:0.855 lr:9.9e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 16:48:39 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:48:39 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:49:33 in_model.py:236 step:2K smpl:24K ep:183 epch:9.16 loss:0.045 grdn:0.835 lr:9.9e-05 updt_s:1.050 data_s:0.021
+WARNING 2025-11-06 16:49:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:49:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:50:26 in_model.py:236 step:2K smpl:25K ep:189 epch:9.46 loss:0.043 grdn:0.803 lr:9.9e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 16:50:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:50:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:51:19 in_model.py:236 step:2K smpl:26K ep:195 epch:9.77 loss:0.047 grdn:0.870 lr:9.9e-05 updt_s:1.060 data_s:0.000
+WARNING 2025-11-06 16:51:19 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:51:19 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:52:13 in_model.py:236 step:2K smpl:26K ep:201 epch:10.07 loss:0.042 grdn:0.784 lr:9.9e-05 updt_s:1.057 data_s:0.024
+WARNING 2025-11-06 16:52:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:52:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:53:06 in_model.py:236 step:2K smpl:27K ep:208 epch:10.38 loss:0.043 grdn:0.775 lr:9.9e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 16:53:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:53:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:53:59 in_model.py:236 step:2K smpl:28K ep:214 epch:10.68 loss:0.042 grdn:0.799 lr:9.9e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 16:53:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:53:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:54:52 in_model.py:236 step:2K smpl:29K ep:220 epch:10.99 loss:0.038 grdn:0.714 lr:9.9e-05 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 16:54:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:54:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:55:45 in_model.py:236 step:2K smpl:30K ep:226 epch:11.29 loss:0.040 grdn:0.753 lr:9.9e-05 updt_s:1.049 data_s:0.021
+WARNING 2025-11-06 16:55:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:55:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:56:38 in_model.py:236 step:2K smpl:30K ep:232 epch:11.60 loss:0.038 grdn:0.699 lr:9.9e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 16:56:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:56:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:57:31 in_model.py:236 step:2K smpl:31K ep:238 epch:11.90 loss:0.038 grdn:0.660 lr:9.9e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 16:57:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:57:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:58:25 in_model.py:236 step:2K smpl:32K ep:244 epch:12.21 loss:0.034 grdn:0.633 lr:9.9e-05 updt_s:1.048 data_s:0.021
+WARNING 2025-11-06 16:58:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:58:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 16:59:17 in_model.py:236 step:2K smpl:33K ep:250 epch:12.51 loss:0.039 grdn:0.733 lr:9.9e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 16:59:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 16:59:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:00:10 in_model.py:236 step:2K smpl:34K ep:256 epch:12.82 loss:0.037 grdn:0.686 lr:9.9e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:00:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:00:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:01:04 in_model.py:236 step:2K smpl:34K ep:262 epch:13.12 loss:0.038 grdn:0.758 lr:9.9e-05 updt_s:1.048 data_s:0.021
+WARNING 2025-11-06 17:01:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:01:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:01:57 in_model.py:236 step:2K smpl:35K ep:269 epch:13.43 loss:0.034 grdn:0.704 lr:9.9e-05 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 17:01:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:01:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:02:50 in_model.py:236 step:2K smpl:36K ep:275 epch:13.74 loss:0.030 grdn:0.661 lr:9.9e-05 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 17:02:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:02:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:03:43 in_model.py:236 step:2K smpl:37K ep:281 epch:14.04 loss:0.036 grdn:0.715 lr:9.9e-05 updt_s:1.048 data_s:0.022
+WARNING 2025-11-06 17:03:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:03:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:04:36 in_model.py:236 step:2K smpl:38K ep:287 epch:14.35 loss:0.039 grdn:0.753 lr:9.9e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:04:36 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:04:36 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:05:30 in_model.py:236 step:2K smpl:38K ep:293 epch:14.65 loss:0.035 grdn:0.735 lr:9.8e-05 updt_s:1.063 data_s:0.000
+WARNING 2025-11-06 17:05:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:05:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:06:23 in_model.py:236 step:2K smpl:39K ep:299 epch:14.96 loss:0.031 grdn:0.641 lr:9.8e-05 updt_s:1.064 data_s:0.000
+WARNING 2025-11-06 17:06:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:06:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:07:17 in_model.py:236 step:2K smpl:40K ep:305 epch:15.26 loss:0.038 grdn:0.796 lr:9.8e-05 updt_s:1.059 data_s:0.020
+WARNING 2025-11-06 17:07:17 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:07:17 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:08:10 in_model.py:236 step:3K smpl:41K ep:311 epch:15.57 loss:0.036 grdn:0.678 lr:9.8e-05 updt_s:1.058 data_s:0.000
+WARNING 2025-11-06 17:08:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:08:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:09:03 in_model.py:236 step:3K smpl:42K ep:317 epch:15.87 loss:0.036 grdn:0.720 lr:9.8e-05 updt_s:1.063 data_s:0.000
+WARNING 2025-11-06 17:09:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:09:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:09:57 in_model.py:236 step:3K smpl:42K ep:324 epch:16.18 loss:0.028 grdn:0.588 lr:9.8e-05 updt_s:1.059 data_s:0.021
+WARNING 2025-11-06 17:09:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:09:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:10:51 in_model.py:236 step:3K smpl:43K ep:330 epch:16.48 loss:0.029 grdn:0.607 lr:9.8e-05 updt_s:1.060 data_s:0.000
+WARNING 2025-11-06 17:10:51 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:10:51 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:11:44 in_model.py:236 step:3K smpl:44K ep:336 epch:16.79 loss:0.029 grdn:0.611 lr:9.8e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 17:11:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:11:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:12:37 in_model.py:236 step:3K smpl:45K ep:342 epch:17.09 loss:0.031 grdn:0.626 lr:9.8e-05 updt_s:1.049 data_s:0.020
+WARNING 2025-11-06 17:12:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:12:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:13:30 in_model.py:236 step:3K smpl:46K ep:348 epch:17.40 loss:0.030 grdn:0.638 lr:9.8e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:13:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:13:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:14:23 in_model.py:236 step:3K smpl:46K ep:354 epch:17.70 loss:0.035 grdn:0.653 lr:9.8e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 17:14:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:14:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:15:16 in_model.py:236 step:3K smpl:47K ep:360 epch:18.01 loss:0.036 grdn:0.653 lr:9.8e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 17:15:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:15:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:16:09 in_model.py:236 step:3K smpl:48K ep:366 epch:18.31 loss:0.025 grdn:0.577 lr:9.8e-05 updt_s:1.049 data_s:0.021
+WARNING 2025-11-06 17:16:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:16:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:17:02 in_model.py:236 step:3K smpl:49K ep:372 epch:18.62 loss:0.036 grdn:0.641 lr:9.8e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:17:02 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:17:02 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:17:55 in_model.py:236 step:3K smpl:50K ep:378 epch:18.92 loss:0.028 grdn:0.591 lr:9.7e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:17:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:17:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:18:49 in_model.py:236 step:3K smpl:50K ep:385 epch:19.23 loss:0.027 grdn:0.589 lr:9.7e-05 updt_s:1.049 data_s:0.020
+WARNING 2025-11-06 17:18:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:18:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:19:42 in_model.py:236 step:3K smpl:51K ep:391 epch:19.53 loss:0.027 grdn:0.577 lr:9.7e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 17:19:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:19:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:20:35 in_model.py:236 step:3K smpl:52K ep:397 epch:19.84 loss:0.028 grdn:0.567 lr:9.7e-05 updt_s:1.058 data_s:0.000
+WARNING 2025-11-06 17:20:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:20:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:21:28 in_model.py:236 step:3K smpl:53K ep:403 epch:20.14 loss:0.029 grdn:0.603 lr:9.7e-05 updt_s:1.049 data_s:0.020
+WARNING 2025-11-06 17:21:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:21:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:22:21 in_model.py:236 step:3K smpl:54K ep:409 epch:20.45 loss:0.030 grdn:0.611 lr:9.7e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:22:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:22:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:23:14 in_model.py:236 step:3K smpl:54K ep:415 epch:20.76 loss:0.025 grdn:0.572 lr:9.7e-05 updt_s:1.057 data_s:0.000
+WARNING 2025-11-06 17:23:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:23:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:24:08 in_model.py:236 step:3K smpl:55K ep:421 epch:21.06 loss:0.030 grdn:0.592 lr:9.7e-05 updt_s:1.050 data_s:0.021
+WARNING 2025-11-06 17:24:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:24:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:25:01 in_model.py:236 step:4K smpl:56K ep:427 epch:21.37 loss:0.029 grdn:0.615 lr:9.7e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:25:01 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:25:01 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:25:54 in_model.py:236 step:4K smpl:57K ep:433 epch:21.67 loss:0.025 grdn:0.524 lr:9.7e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:25:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:25:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:26:47 in_model.py:236 step:4K smpl:58K ep:440 epch:21.98 loss:0.032 grdn:0.574 lr:9.7e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 17:26:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:26:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:27:40 in_model.py:236 step:4K smpl:58K ep:446 epch:22.28 loss:0.025 grdn:0.527 lr:9.7e-05 updt_s:1.045 data_s:0.022
+WARNING 2025-11-06 17:27:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:27:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:28:33 in_model.py:236 step:4K smpl:59K ep:452 epch:22.59 loss:0.030 grdn:0.578 lr:9.6e-05 updt_s:1.046 data_s:0.000
+WARNING 2025-11-06 17:28:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:28:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:29:25 in_model.py:236 step:4K smpl:60K ep:458 epch:22.89 loss:0.025 grdn:0.504 lr:9.6e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:29:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:29:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:30:19 in_model.py:236 step:4K smpl:61K ep:464 epch:23.20 loss:0.030 grdn:0.586 lr:9.6e-05 updt_s:1.048 data_s:0.021
+WARNING 2025-11-06 17:30:19 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:30:19 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:31:12 in_model.py:236 step:4K smpl:62K ep:470 epch:23.50 loss:0.032 grdn:0.626 lr:9.6e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:31:12 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:31:12 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:32:05 in_model.py:236 step:4K smpl:62K ep:476 epch:23.81 loss:0.032 grdn:0.594 lr:9.6e-05 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 17:32:05 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:32:05 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:32:58 in_model.py:236 step:4K smpl:63K ep:482 epch:24.11 loss:0.025 grdn:0.536 lr:9.6e-05 updt_s:1.045 data_s:0.022
+WARNING 2025-11-06 17:32:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:32:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:33:51 in_model.py:236 step:4K smpl:64K ep:488 epch:24.42 loss:0.025 grdn:0.529 lr:9.6e-05 updt_s:1.049 data_s:0.000
+WARNING 2025-11-06 17:33:51 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:33:51 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:34:44 in_model.py:236 step:4K smpl:65K ep:494 epch:24.72 loss:0.024 grdn:0.528 lr:9.6e-05 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 17:34:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:34:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:35:36 in_model.py:236 step:4K smpl:66K ep:501 epch:25.03 loss:0.026 grdn:0.545 lr:9.6e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 17:35:36 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:35:36 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:36:30 in_model.py:236 step:4K smpl:66K ep:507 epch:25.33 loss:0.027 grdn:0.549 lr:9.6e-05 updt_s:1.052 data_s:0.021
+WARNING 2025-11-06 17:36:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:36:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:37:23 in_model.py:236 step:4K smpl:67K ep:513 epch:25.64 loss:0.025 grdn:0.554 lr:9.5e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:37:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:37:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:38:16 in_model.py:236 step:4K smpl:68K ep:519 epch:25.94 loss:0.023 grdn:0.522 lr:9.5e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:38:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:38:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:39:10 in_model.py:236 step:4K smpl:69K ep:525 epch:26.25 loss:0.025 grdn:0.564 lr:9.5e-05 updt_s:1.050 data_s:0.020
+WARNING 2025-11-06 17:39:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:39:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:40:03 in_model.py:236 step:4K smpl:70K ep:531 epch:26.55 loss:0.027 grdn:0.561 lr:9.5e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:40:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:40:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:40:55 in_model.py:236 step:4K smpl:70K ep:537 epch:26.86 loss:0.027 grdn:0.616 lr:9.5e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 17:40:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:40:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:41:49 in_model.py:236 step:4K smpl:71K ep:543 epch:27.17 loss:0.030 grdn:0.585 lr:9.5e-05 updt_s:1.047 data_s:0.021
+WARNING 2025-11-06 17:41:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:41:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:42:42 in_model.py:236 step:4K smpl:72K ep:549 epch:27.47 loss:0.023 grdn:0.508 lr:9.5e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 17:42:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:42:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:43:35 in_model.py:236 step:5K smpl:73K ep:556 epch:27.78 loss:0.023 grdn:0.472 lr:9.5e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:43:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:43:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:44:28 in_model.py:236 step:5K smpl:74K ep:562 epch:28.08 loss:0.025 grdn:0.559 lr:9.5e-05 updt_s:1.048 data_s:0.021
+WARNING 2025-11-06 17:44:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:44:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:45:21 in_model.py:236 step:5K smpl:74K ep:568 epch:28.39 loss:0.023 grdn:0.552 lr:9.4e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:45:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:45:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:46:14 in_model.py:236 step:5K smpl:75K ep:574 epch:28.69 loss:0.027 grdn:0.555 lr:9.4e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:46:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:46:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:47:07 in_model.py:236 step:5K smpl:76K ep:580 epch:29.00 loss:0.025 grdn:0.503 lr:9.4e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:47:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:47:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:48:00 in_model.py:236 step:5K smpl:77K ep:586 epch:29.30 loss:0.024 grdn:0.523 lr:9.4e-05 updt_s:1.045 data_s:0.020
+WARNING 2025-11-06 17:48:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:48:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:48:53 in_model.py:236 step:5K smpl:78K ep:592 epch:29.61 loss:0.023 grdn:0.479 lr:9.4e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 17:48:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:48:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:49:46 in_model.py:236 step:5K smpl:78K ep:598 epch:29.91 loss:0.026 grdn:0.566 lr:9.4e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 17:49:46 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:49:46 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:50:40 in_model.py:236 step:5K smpl:79K ep:604 epch:30.22 loss:0.021 grdn:0.490 lr:9.4e-05 updt_s:1.051 data_s:0.020
+WARNING 2025-11-06 17:50:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:50:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:51:33 in_model.py:236 step:5K smpl:80K ep:610 epch:30.52 loss:0.021 grdn:0.485 lr:9.4e-05 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 17:51:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:51:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:51:33 in_model.py:245 Checkpoint policy after step 5000
+INFO 2025-11-06 17:52:29 in_model.py:236 step:5K smpl:81K ep:617 epch:30.83 loss:0.022 grdn:0.503 lr:9.3e-05 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 17:52:29 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:52:29 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:53:23 in_model.py:236 step:5K smpl:82K ep:623 epch:31.13 loss:0.025 grdn:0.549 lr:9.3e-05 updt_s:1.058 data_s:0.022
+WARNING 2025-11-06 17:53:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:53:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:54:17 in_model.py:236 step:5K smpl:82K ep:629 epch:31.44 loss:0.022 grdn:0.462 lr:9.3e-05 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 17:54:17 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:54:17 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:55:10 in_model.py:236 step:5K smpl:83K ep:635 epch:31.74 loss:0.020 grdn:0.458 lr:9.3e-05 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 17:55:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:55:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:56:04 in_model.py:236 step:5K smpl:84K ep:641 epch:32.05 loss:0.019 grdn:0.441 lr:9.3e-05 updt_s:1.059 data_s:0.023
+WARNING 2025-11-06 17:56:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:56:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:56:57 in_model.py:236 step:5K smpl:85K ep:647 epch:32.35 loss:0.023 grdn:0.502 lr:9.3e-05 updt_s:1.065 data_s:0.000
+WARNING 2025-11-06 17:56:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:56:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:57:51 in_model.py:236 step:5K smpl:86K ep:653 epch:32.66 loss:0.024 grdn:0.487 lr:9.3e-05 updt_s:1.069 data_s:0.000
+WARNING 2025-11-06 17:57:51 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:57:51 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:58:45 in_model.py:236 step:5K smpl:86K ep:659 epch:32.96 loss:0.020 grdn:0.441 lr:9.2e-05 updt_s:1.073 data_s:0.000
+WARNING 2025-11-06 17:58:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:58:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 17:59:39 in_model.py:236 step:5K smpl:87K ep:665 epch:33.27 loss:0.024 grdn:0.523 lr:9.2e-05 updt_s:1.064 data_s:0.021
+WARNING 2025-11-06 17:59:39 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 17:59:39 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:00:33 in_model.py:236 step:6K smpl:88K ep:671 epch:33.57 loss:0.022 grdn:0.471 lr:9.2e-05 updt_s:1.072 data_s:0.000
+WARNING 2025-11-06 18:00:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:00:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:01:27 in_model.py:236 step:6K smpl:89K ep:678 epch:33.88 loss:0.021 grdn:0.422 lr:9.2e-05 updt_s:1.071 data_s:0.000
+WARNING 2025-11-06 18:01:27 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:01:27 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:02:21 in_model.py:236 step:6K smpl:90K ep:684 epch:34.19 loss:0.025 grdn:0.555 lr:9.2e-05 updt_s:1.061 data_s:0.022
+WARNING 2025-11-06 18:02:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:02:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:03:15 in_model.py:236 step:6K smpl:90K ep:690 epch:34.49 loss:0.020 grdn:0.464 lr:9.2e-05 updt_s:1.068 data_s:0.000
+WARNING 2025-11-06 18:03:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:03:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:04:08 in_model.py:236 step:6K smpl:91K ep:696 epch:34.80 loss:0.023 grdn:0.500 lr:9.2e-05 updt_s:1.066 data_s:0.000
+WARNING 2025-11-06 18:04:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:04:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:05:02 in_model.py:236 step:6K smpl:92K ep:702 epch:35.10 loss:0.021 grdn:0.453 lr:9.1e-05 updt_s:1.060 data_s:0.022
+WARNING 2025-11-06 18:05:02 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:05:02 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:05:56 in_model.py:236 step:6K smpl:93K ep:708 epch:35.41 loss:0.022 grdn:0.518 lr:9.1e-05 updt_s:1.072 data_s:0.000
+WARNING 2025-11-06 18:05:56 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:05:56 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:06:50 in_model.py:236 step:6K smpl:94K ep:714 epch:35.71 loss:0.021 grdn:0.456 lr:9.1e-05 updt_s:1.081 data_s:0.000
+WARNING 2025-11-06 18:06:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:06:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:07:44 in_model.py:236 step:6K smpl:94K ep:720 epch:36.02 loss:0.020 grdn:0.463 lr:9.1e-05 updt_s:1.082 data_s:0.000
+WARNING 2025-11-06 18:07:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:07:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:08:39 in_model.py:236 step:6K smpl:95K ep:726 epch:36.32 loss:0.024 grdn:0.583 lr:9.1e-05 updt_s:1.066 data_s:0.021
+WARNING 2025-11-06 18:08:39 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:08:39 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:09:32 in_model.py:236 step:6K smpl:96K ep:733 epch:36.63 loss:0.019 grdn:0.503 lr:9.1e-05 updt_s:1.062 data_s:0.000
+WARNING 2025-11-06 18:09:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:09:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:10:26 in_model.py:236 step:6K smpl:97K ep:739 epch:36.93 loss:0.021 grdn:0.497 lr:9.1e-05 updt_s:1.064 data_s:0.000
+WARNING 2025-11-06 18:10:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:10:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:11:20 in_model.py:236 step:6K smpl:98K ep:745 epch:37.24 loss:0.020 grdn:0.467 lr:9.0e-05 updt_s:1.056 data_s:0.022
+WARNING 2025-11-06 18:11:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:11:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:12:13 in_model.py:236 step:6K smpl:98K ep:751 epch:37.54 loss:0.019 grdn:0.423 lr:9.0e-05 updt_s:1.058 data_s:0.000
+WARNING 2025-11-06 18:12:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:12:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:13:06 in_model.py:236 step:6K smpl:99K ep:757 epch:37.85 loss:0.020 grdn:0.476 lr:9.0e-05 updt_s:1.059 data_s:0.000
+WARNING 2025-11-06 18:13:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:13:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:13:59 in_model.py:236 step:6K smpl:100K ep:763 epch:38.15 loss:0.024 grdn:0.498 lr:9.0e-05 updt_s:1.044 data_s:0.022
+WARNING 2025-11-06 18:13:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:13:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:14:52 in_model.py:236 step:6K smpl:101K ep:769 epch:38.46 loss:0.025 grdn:0.478 lr:9.0e-05 updt_s:1.051 data_s:0.000
+WARNING 2025-11-06 18:14:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:14:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:15:44 in_model.py:236 step:6K smpl:102K ep:775 epch:38.76 loss:0.020 grdn:0.436 lr:9.0e-05 updt_s:1.049 data_s:0.000
+WARNING 2025-11-06 18:15:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:15:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:16:38 in_model.py:236 step:6K smpl:102K ep:781 epch:39.07 loss:0.020 grdn:0.487 lr:9.0e-05 updt_s:1.049 data_s:0.022
+WARNING 2025-11-06 18:16:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:16:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:17:30 in_model.py:236 step:6K smpl:103K ep:787 epch:39.37 loss:0.020 grdn:0.449 lr:8.9e-05 updt_s:1.041 data_s:0.000
+WARNING 2025-11-06 18:17:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:17:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:18:23 in_model.py:236 step:6K smpl:104K ep:794 epch:39.68 loss:0.018 grdn:0.453 lr:8.9e-05 updt_s:1.047 data_s:0.000
+WARNING 2025-11-06 18:18:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:18:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:19:15 in_model.py:236 step:7K smpl:105K ep:800 epch:39.98 loss:0.020 grdn:0.454 lr:8.9e-05 updt_s:1.045 data_s:0.000
+WARNING 2025-11-06 18:19:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:19:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:20:09 in_model.py:236 step:7K smpl:106K ep:806 epch:40.29 loss:0.023 grdn:0.511 lr:8.9e-05 updt_s:1.043 data_s:0.021
+WARNING 2025-11-06 18:20:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:20:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:21:01 in_model.py:236 step:7K smpl:106K ep:812 epch:40.60 loss:0.018 grdn:0.436 lr:8.9e-05 updt_s:1.046 data_s:0.000
+WARNING 2025-11-06 18:21:01 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:21:01 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:21:53 in_model.py:236 step:7K smpl:107K ep:818 epch:40.90 loss:0.021 grdn:0.485 lr:8.9e-05 updt_s:1.038 data_s:0.000
+WARNING 2025-11-06 18:21:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:21:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:22:46 in_model.py:236 step:7K smpl:108K ep:824 epch:41.21 loss:0.020 grdn:0.435 lr:8.8e-05 updt_s:1.035 data_s:0.021
+WARNING 2025-11-06 18:22:46 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:22:46 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:23:38 in_model.py:236 step:7K smpl:109K ep:830 epch:41.51 loss:0.023 grdn:0.482 lr:8.8e-05 updt_s:1.039 data_s:0.000
+WARNING 2025-11-06 18:23:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:23:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:24:30 in_model.py:236 step:7K smpl:110K ep:836 epch:41.82 loss:0.021 grdn:0.454 lr:8.8e-05 updt_s:1.040 data_s:0.000
+WARNING 2025-11-06 18:24:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:24:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:25:23 in_model.py:236 step:7K smpl:110K ep:842 epch:42.12 loss:0.021 grdn:0.461 lr:8.8e-05 updt_s:1.031 data_s:0.021
+WARNING 2025-11-06 18:25:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:25:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:26:15 in_model.py:236 step:7K smpl:111K ep:849 epch:42.43 loss:0.021 grdn:0.455 lr:8.8e-05 updt_s:1.038 data_s:0.000
+WARNING 2025-11-06 18:26:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:26:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:27:07 in_model.py:236 step:7K smpl:112K ep:855 epch:42.73 loss:0.020 grdn:0.468 lr:8.8e-05 updt_s:1.036 data_s:0.000
+WARNING 2025-11-06 18:27:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:27:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:27:59 in_model.py:236 step:7K smpl:113K ep:861 epch:43.04 loss:0.020 grdn:0.423 lr:8.7e-05 updt_s:1.038 data_s:0.000
+WARNING 2025-11-06 18:27:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:27:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:28:52 in_model.py:236 step:7K smpl:114K ep:867 epch:43.34 loss:0.022 grdn:0.467 lr:8.7e-05 updt_s:1.032 data_s:0.020
+WARNING 2025-11-06 18:28:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:28:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:29:44 in_model.py:236 step:7K smpl:114K ep:873 epch:43.65 loss:0.021 grdn:0.453 lr:8.7e-05 updt_s:1.038 data_s:0.000
+WARNING 2025-11-06 18:29:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:29:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:30:36 in_model.py:236 step:7K smpl:115K ep:879 epch:43.95 loss:0.016 grdn:0.421 lr:8.7e-05 updt_s:1.036 data_s:0.000
+WARNING 2025-11-06 18:30:36 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:30:36 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:31:28 in_model.py:236 step:7K smpl:116K ep:885 epch:44.26 loss:0.020 grdn:0.437 lr:8.7e-05 updt_s:1.020 data_s:0.021
+WARNING 2025-11-06 18:31:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:31:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:32:20 in_model.py:236 step:7K smpl:117K ep:891 epch:44.56 loss:0.020 grdn:0.457 lr:8.7e-05 updt_s:1.025 data_s:0.000
+WARNING 2025-11-06 18:32:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:32:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:33:11 in_model.py:236 step:7K smpl:118K ep:897 epch:44.87 loss:0.023 grdn:0.471 lr:8.6e-05 updt_s:1.028 data_s:0.000
+WARNING 2025-11-06 18:33:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:33:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:34:04 in_model.py:236 step:7K smpl:118K ep:903 epch:45.17 loss:0.020 grdn:0.456 lr:8.6e-05 updt_s:1.022 data_s:0.022
+WARNING 2025-11-06 18:34:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:34:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:34:55 in_model.py:236 step:7K smpl:119K ep:910 epch:45.48 loss:0.017 grdn:0.464 lr:8.6e-05 updt_s:1.029 data_s:0.000
+WARNING 2025-11-06 18:34:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:34:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:35:47 in_model.py:236 step:8K smpl:120K ep:916 epch:45.78 loss:0.019 grdn:0.429 lr:8.6e-05 updt_s:1.032 data_s:0.000
+WARNING 2025-11-06 18:35:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:35:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:36:39 in_model.py:236 step:8K smpl:121K ep:922 epch:46.09 loss:0.020 grdn:0.434 lr:8.6e-05 updt_s:1.024 data_s:0.022
+WARNING 2025-11-06 18:36:39 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:36:39 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:37:31 in_model.py:236 step:8K smpl:122K ep:928 epch:46.39 loss:0.021 grdn:0.469 lr:8.5e-05 updt_s:1.032 data_s:0.000
+WARNING 2025-11-06 18:37:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:37:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:38:23 in_model.py:236 step:8K smpl:122K ep:934 epch:46.70 loss:0.018 grdn:0.417 lr:8.5e-05 updt_s:1.031 data_s:0.000
+WARNING 2025-11-06 18:38:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:38:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:39:14 in_model.py:236 step:8K smpl:123K ep:940 epch:47.00 loss:0.017 grdn:0.450 lr:8.5e-05 updt_s:1.027 data_s:0.000
+WARNING 2025-11-06 18:39:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:39:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:40:07 in_model.py:236 step:8K smpl:124K ep:946 epch:47.31 loss:0.019 grdn:0.491 lr:8.5e-05 updt_s:1.019 data_s:0.022
+WARNING 2025-11-06 18:40:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:40:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:40:58 in_model.py:236 step:8K smpl:125K ep:952 epch:47.62 loss:0.018 grdn:0.401 lr:8.5e-05 updt_s:1.019 data_s:0.000
+WARNING 2025-11-06 18:40:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:40:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:41:49 in_model.py:236 step:8K smpl:126K ep:958 epch:47.92 loss:0.017 grdn:0.428 lr:8.5e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 18:41:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:41:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:42:41 in_model.py:236 step:8K smpl:126K ep:965 epch:48.23 loss:0.018 grdn:0.441 lr:8.4e-05 updt_s:1.015 data_s:0.021
+WARNING 2025-11-06 18:42:41 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:42:41 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:43:32 in_model.py:236 step:8K smpl:127K ep:971 epch:48.53 loss:0.017 grdn:0.438 lr:8.4e-05 updt_s:1.025 data_s:0.000
+WARNING 2025-11-06 18:43:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:43:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:44:24 in_model.py:236 step:8K smpl:128K ep:977 epch:48.84 loss:0.017 grdn:0.396 lr:8.4e-05 updt_s:1.024 data_s:0.000
+WARNING 2025-11-06 18:44:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:44:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:45:16 in_model.py:236 step:8K smpl:129K ep:983 epch:49.14 loss:0.022 grdn:0.510 lr:8.4e-05 updt_s:1.018 data_s:0.022
+WARNING 2025-11-06 18:45:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:45:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:46:07 in_model.py:236 step:8K smpl:130K ep:989 epch:49.45 loss:0.015 grdn:0.357 lr:8.4e-05 updt_s:1.025 data_s:0.000
+WARNING 2025-11-06 18:46:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:46:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:46:58 in_model.py:236 step:8K smpl:130K ep:995 epch:49.75 loss:0.018 grdn:0.442 lr:8.3e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 18:46:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:46:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:47:50 in_model.py:236 step:8K smpl:131K ep:1K epch:50.06 loss:0.020 grdn:0.416 lr:8.3e-05 updt_s:1.020 data_s:0.000
+WARNING 2025-11-06 18:47:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:47:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:48:42 in_model.py:236 step:8K smpl:132K ep:1K epch:50.36 loss:0.019 grdn:0.433 lr:8.3e-05 updt_s:1.019 data_s:0.021
+WARNING 2025-11-06 18:48:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:48:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:49:33 in_model.py:236 step:8K smpl:133K ep:1K epch:50.67 loss:0.018 grdn:0.458 lr:8.3e-05 updt_s:1.020 data_s:0.000
+WARNING 2025-11-06 18:49:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:49:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:50:24 in_model.py:236 step:8K smpl:134K ep:1K epch:50.97 loss:0.016 grdn:0.340 lr:8.3e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 18:50:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:50:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:51:16 in_model.py:236 step:8K smpl:134K ep:1K epch:51.28 loss:0.018 grdn:0.419 lr:8.2e-05 updt_s:1.014 data_s:0.020
+WARNING 2025-11-06 18:51:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:51:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:52:07 in_model.py:236 step:8K smpl:135K ep:1K epch:51.58 loss:0.019 grdn:0.436 lr:8.2e-05 updt_s:1.019 data_s:0.000
+WARNING 2025-11-06 18:52:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:52:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:52:59 in_model.py:236 step:8K smpl:136K ep:1K epch:51.89 loss:0.016 grdn:0.383 lr:8.2e-05 updt_s:1.020 data_s:0.000
+WARNING 2025-11-06 18:52:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:52:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:53:50 in_model.py:236 step:9K smpl:137K ep:1K epch:52.19 loss:0.019 grdn:0.446 lr:8.2e-05 updt_s:1.012 data_s:0.022
+WARNING 2025-11-06 18:53:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:53:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:54:42 in_model.py:236 step:9K smpl:138K ep:1K epch:52.50 loss:0.018 grdn:0.429 lr:8.2e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 18:54:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:54:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:55:33 in_model.py:236 step:9K smpl:138K ep:1K epch:52.80 loss:0.016 grdn:0.401 lr:8.1e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 18:55:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:55:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:56:25 in_model.py:236 step:9K smpl:139K ep:1K epch:53.11 loss:0.018 grdn:0.394 lr:8.1e-05 updt_s:1.015 data_s:0.020
+WARNING 2025-11-06 18:56:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:56:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:57:16 in_model.py:236 step:9K smpl:140K ep:1K epch:53.41 loss:0.018 grdn:0.447 lr:8.1e-05 updt_s:1.018 data_s:0.000
+WARNING 2025-11-06 18:57:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:57:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:58:07 in_model.py:236 step:9K smpl:141K ep:1K epch:53.72 loss:0.018 grdn:0.415 lr:8.1e-05 updt_s:1.021 data_s:0.000
+WARNING 2025-11-06 18:58:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:58:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:58:58 in_model.py:236 step:9K smpl:142K ep:1K epch:54.03 loss:0.016 grdn:0.393 lr:8.1e-05 updt_s:1.021 data_s:0.000
+WARNING 2025-11-06 18:58:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:58:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 18:59:50 in_model.py:236 step:9K smpl:142K ep:1K epch:54.33 loss:0.021 grdn:0.461 lr:8.0e-05 updt_s:1.016 data_s:0.022
+WARNING 2025-11-06 18:59:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 18:59:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:00:42 in_model.py:236 step:9K smpl:143K ep:1K epch:54.64 loss:0.017 grdn:0.391 lr:8.0e-05 updt_s:1.026 data_s:0.000
+WARNING 2025-11-06 19:00:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:00:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:01:33 in_model.py:236 step:9K smpl:144K ep:1K epch:54.94 loss:0.015 grdn:0.400 lr:8.0e-05 updt_s:1.025 data_s:0.000
+WARNING 2025-11-06 19:01:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:01:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:02:25 in_model.py:236 step:9K smpl:145K ep:1K epch:55.25 loss:0.017 grdn:0.433 lr:8.0e-05 updt_s:1.017 data_s:0.021
+WARNING 2025-11-06 19:02:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:02:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:03:17 in_model.py:236 step:9K smpl:146K ep:1K epch:55.55 loss:0.016 grdn:0.401 lr:8.0e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 19:03:17 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:03:17 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:04:08 in_model.py:236 step:9K smpl:146K ep:1K epch:55.86 loss:0.019 grdn:0.424 lr:7.9e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 19:04:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:04:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:05:00 in_model.py:236 step:9K smpl:147K ep:1K epch:56.16 loss:0.016 grdn:0.393 lr:7.9e-05 updt_s:1.017 data_s:0.021
+WARNING 2025-11-06 19:05:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:05:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:05:51 in_model.py:236 step:9K smpl:148K ep:1K epch:56.47 loss:0.020 grdn:0.404 lr:7.9e-05 updt_s:1.019 data_s:0.000
+WARNING 2025-11-06 19:05:51 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:05:51 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:06:42 in_model.py:236 step:9K smpl:149K ep:1K epch:56.77 loss:0.015 grdn:0.359 lr:7.9e-05 updt_s:1.017 data_s:0.000
+WARNING 2025-11-06 19:06:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:06:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:07:34 in_model.py:236 step:9K smpl:150K ep:1K epch:57.08 loss:0.016 grdn:0.374 lr:7.9e-05 updt_s:1.009 data_s:0.020
+WARNING 2025-11-06 19:07:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:07:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:08:25 in_model.py:236 step:9K smpl:150K ep:1K epch:57.38 loss:0.015 grdn:0.369 lr:7.8e-05 updt_s:1.024 data_s:0.000
+WARNING 2025-11-06 19:08:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:08:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:09:17 in_model.py:236 step:9K smpl:151K ep:1K epch:57.69 loss:0.016 grdn:0.401 lr:7.8e-05 updt_s:1.038 data_s:0.000
+WARNING 2025-11-06 19:09:17 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:09:17 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:10:10 in_model.py:236 step:10K smpl:152K ep:1K epch:57.99 loss:0.018 grdn:0.440 lr:7.8e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 19:10:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:10:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:11:04 in_model.py:236 step:10K smpl:153K ep:1K epch:58.30 loss:0.013 grdn:0.358 lr:7.8e-05 updt_s:1.051 data_s:0.021
+WARNING 2025-11-06 19:11:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:11:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:11:57 in_model.py:236 step:10K smpl:154K ep:1K epch:58.60 loss:0.015 grdn:0.359 lr:7.7e-05 updt_s:1.062 data_s:0.000
+WARNING 2025-11-06 19:11:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:11:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:12:50 in_model.py:236 step:10K smpl:154K ep:1K epch:58.91 loss:0.015 grdn:0.406 lr:7.7e-05 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 19:12:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:12:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:13:44 in_model.py:236 step:10K smpl:155K ep:1K epch:59.21 loss:0.014 grdn:0.357 lr:7.7e-05 updt_s:1.055 data_s:0.020
+WARNING 2025-11-06 19:13:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:13:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:14:37 in_model.py:236 step:10K smpl:156K ep:1K epch:59.52 loss:0.018 grdn:0.424 lr:7.7e-05 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 19:14:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:14:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:15:30 in_model.py:236 step:10K smpl:157K ep:1K epch:59.82 loss:0.017 grdn:0.395 lr:7.7e-05 updt_s:1.062 data_s:0.000
+WARNING 2025-11-06 19:15:30 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:15:30 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:16:24 in_model.py:236 step:10K smpl:158K ep:1K epch:60.13 loss:0.017 grdn:0.400 lr:7.6e-05 updt_s:1.056 data_s:0.020
+WARNING 2025-11-06 19:16:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:16:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:17:18 in_model.py:236 step:10K smpl:158K ep:1K epch:60.43 loss:0.015 grdn:0.362 lr:7.6e-05 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 19:17:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:17:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:18:11 in_model.py:236 step:10K smpl:159K ep:1K epch:60.74 loss:0.016 grdn:0.421 lr:7.6e-05 updt_s:1.064 data_s:0.000
+WARNING 2025-11-06 19:18:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:18:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:19:04 in_model.py:236 step:10K smpl:160K ep:1K epch:61.05 loss:0.015 grdn:0.381 lr:7.6e-05 updt_s:1.064 data_s:0.000
+WARNING 2025-11-06 19:19:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:19:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:19:04 in_model.py:245 Checkpoint policy after step 10000
+INFO 2025-11-06 19:20:01 in_model.py:236 step:10K smpl:161K ep:1K epch:61.35 loss:0.015 grdn:0.362 lr:7.6e-05 updt_s:1.044 data_s:0.023
+WARNING 2025-11-06 19:20:01 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:20:01 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:20:54 in_model.py:236 step:10K smpl:162K ep:1K epch:61.66 loss:0.016 grdn:0.378 lr:7.5e-05 updt_s:1.061 data_s:0.000
+WARNING 2025-11-06 19:20:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:20:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:21:47 in_model.py:236 step:10K smpl:162K ep:1K epch:61.96 loss:0.015 grdn:0.408 lr:7.5e-05 updt_s:1.062 data_s:0.000
+WARNING 2025-11-06 19:21:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:21:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:22:41 in_model.py:236 step:10K smpl:163K ep:1K epch:62.27 loss:0.019 grdn:0.436 lr:7.5e-05 updt_s:1.055 data_s:0.020
+WARNING 2025-11-06 19:22:41 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:22:41 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:23:35 in_model.py:236 step:10K smpl:164K ep:1K epch:62.57 loss:0.013 grdn:0.339 lr:7.5e-05 updt_s:1.067 data_s:0.000
+WARNING 2025-11-06 19:23:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:23:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/files/requirements.txt

@@ -0,0 +1,173 @@
+pyparsing==3.2.5
+nvidia-cuda-cupti-cu12==12.4.127
+pip==25.2
+setuptools==80.9.0
+gitdb==4.0.12
+networkx==3.4.2
+propcache==0.4.1
+aiohappyeyeballs==2.6.1
+aiosignal==1.4.0
+Werkzeug==3.1.3
+nvidia-nvtx-cu12==12.4.127
+typing-inspect==0.9.0
+urllib3==2.5.0
+imageio-ffmpeg==0.6.0
+num2words==0.5.14
+PySocks==1.7.1
+datasets==3.4.1
+PyScreeze==1.0.1
+tokenizers==0.21.4
+yarl==1.22.0
+MouseInfo==0.1.3
+mypy_extensions==1.1.0
+pyzmq==27.1.0
+beautifulsoup4==4.14.2
+importlib_metadata==8.7.0
+evdev==1.9.2
+sentry-sdk==2.43.0
+wcwidth==0.2.14
+typing_extensions==4.15.0
+certifi==2025.10.5
+importlib_resources==6.5.2
+hf-xet==1.2.0
+frozenlist==1.8.0
+annotated-types==0.7.0
+PyGetWindow==0.0.9
+draccus==0.10.0
+rerun-sdk==0.26.2
+cmake==4.1.2
+PyAutoGUI==0.9.54
+gdown==5.2.0
+pyarrow==22.0.0
+nvidia-cusparse-cu12==12.3.1.170
+torch==2.6.0
+cycler==0.12.1
+av==16.0.1
+pytz==2025.2
+tzdata==2025.2
+zipp==3.23.0
+prompt_toolkit==3.0.52
+itsdangerous==2.2.0
+lerobot==0.1.0
+smmap==5.0.2
+pyserial==3.5
+charset-normalizer==3.4.4
+numpy==2.2.6
+psutil==7.1.3
+zarr==2.18.3
+matplotlib==3.10.7
+llvmlite==0.45.1
+torchaudio==2.6.0
+huggingface-hub==0.36.0
+MarkupSafe==3.0.3
+packaging==25.0
+PyYAML==6.0.3
+Flask==3.1.2
+PyOpenGL-accelerate==3.1.10
+hf_transfer==0.1.9
+pydantic_core==2.41.4
+pandas==2.3.3
+mergedeep==1.3.4
+h5py==3.15.1
+transformers==4.50.3
+torchcodec==0.8.1
+termcolor==3.2.0
+toml==0.10.2
+imageio==2.37.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+asciitree==0.3.3
+antlr4-python3-runtime==4.9.3
+jsonlines==4.0.0
+scipy==1.15.3
+pyperclip==1.11.0
+pillow==12.0.0
+click==8.3.0
+torchvision==0.21.0
+cffi==2.0.0
+etils==1.13.0
+Farama-Notifications==0.0.4
+nvidia-cufft-cu12==11.2.1.3
+kiwisolver==1.4.9
+triton==3.2.0
+fonttools==4.60.1
+pycparser==2.23
+absl-py==2.3.1
+requests==2.32.5
+six==1.17.0
+nvidia-nccl-cu12==2.21.5
+pytweening==1.2.0
+docopt==0.6.2
+contourpy==1.3.2
+nvidia-cusolver-cu12==11.6.1.9
+pynput==1.8.1
+Jinja2==3.1.6
+einops==0.8.1
+numcodecs==0.13.1
+async-timeout==5.0.1
+blinker==1.9.0
+pydantic==2.12.3
+xxhash==3.6.0
+cloudpickle==3.1.2
+diffusers==0.35.2
+PyOpenGL==3.1.10
+glfw==2.10.0
+soupsieve==2.8
+GitPython==3.1.45
+fsspec==2024.12.0
+aiohttp==3.13.2
+opencv-python-headless==4.12.0.88
+gymnasium==0.29.1
+python3-xlib==0.15
+mpmath==1.3.0
+nvidia-curand-cu12==10.3.5.147
+wandb==0.22.3
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-runtime-cu12==12.4.127
+python-xlib==0.33
+multiprocess==0.70.16
+pfzy==0.3.4
+orderly-set==5.5.0
+nvidia-nvjitlink-cu12==12.4.127
+accelerate==1.11.0
+regex==2025.11.3
+omegaconf==2.3.0
+safetensors==0.5.3
+filelock==3.20.0
+numba==0.62.1
+nvidia-cusparselt-cu12==0.6.2
+dill==0.3.8
+pyyaml-include==1.4.1
+platformdirs==4.5.0
+multidict==6.7.0
+protobuf==6.33.0
+attrs==25.4.0
+inquirerpy==0.3.4
+PyMsgBox==2.0.1
+pymunk==6.11.1
+nvidia-cudnn-cu12==9.1.0.70
+typing-inspection==0.4.2
+fasteners==0.20
+PyRect==0.2.0
+idna==3.11
+deepdiff==8.6.1
+wheel==0.45.1
+sympy==1.13.1
+python-dateutil==2.9.0.post0
+mujoco==3.1.6
+tqdm==4.67.1
+backports.tarfile==1.2.0
+typeguard==4.3.0
+zipp==3.19.2
+inflect==7.3.1
+more-itertools==10.3.0
+tomli==2.0.1
+jaraco.collections==5.1.0
+jaraco.functools==4.0.1
+typing_extensions==4.12.2
+importlib_metadata==8.0.0
+jaraco.text==3.12.1
+platformdirs==4.2.2
+jaraco.context==5.3.0
+wheel==0.45.1
+packaging==24.2
+autocommand==2.2.2

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/files/wandb-metadata.json

@@ -0,0 +1,64 @@
+{
+  "os":  "Linux-6.8.0-51-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2025-11-06T08:21:49.583849Z",
+  "args":  [
+    "--config_path",
+    "smolvla_omy.yaml"
+  ],
+  "program":  "/workspace/liber/embodied_ai/lerobot-mujoco-tutorial/train_model.py",
+  "codePath":  "train_model.py",
+  "codePathLocal":  "train_model.py",
+  "git":  {
+    "remote":  "git@github.com:liberow/lerobot-mujoco-tutorial.git",
+    "commit":  "b11377a0ce7a3c32cefe260c6dce959b0db2bdda"
+  },
+  "root":  "ckpt/smolvla_omy",
+  "host":  "c81ab3b21da2",
+  "executable":  "/opt/conda/envs/lerobot_mujoco_tutorial/bin/python",
+  "cpu_count":  40,
+  "cpu_count_logical":  80,
+  "gpu":  "Tesla V100-SXM2-32GB",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "1676411871232",
+      "used":  "1481895833600"
+    }
+  },
+  "memory":  {
+    "total":  "134767624192"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-b29b46dc-42a3-9fcc-be17-50b8b8972960"
+    },
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-368aa17c-5504-4d63-4df2-339d38a17fe3"
+    },
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-acf7b3cc-f154-087a-3a06-d57e023c09be"
+    },
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-86b115b3-0485-dd33-56f3-385a1140ae36"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "qj4c6zn4buu7s32sv5nl108gvz7rnfa0"
+}

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug-core.log

@@ -0,0 +1,6 @@
+{"time":"2025-11-06T16:21:49.620623718+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpfd2c2eff/port-528220.txt","pid":528220,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-11-06T16:21:49.621744188+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":528220}
+{"time":"2025-11-06T16:21:49.621734866+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-528220-528551-3638943209/socket","Net":"unix"}}
+{"time":"2025-11-06T16:21:49.795665893+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-11-06T16:21:49.810358292+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"8gs9of7r","id":"1(@)"}
+{"time":"2025-11-06T16:21:50.381980279+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"8gs9of7r","id":"1(@)"}

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2025-11-06T16:21:49.810658746+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2025-11-06T16:21:50.381496059+08:00","level":"INFO","msg":"stream: created new stream","id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.381670461+08:00","level":"INFO","msg":"handler: started","stream_id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.38195187+08:00","level":"INFO","msg":"stream: started","id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.381970995+08:00","level":"INFO","msg":"writer: started","stream_id":"8gs9of7r"}
+{"time":"2025-11-06T16:21:50.381975054+08:00","level":"INFO","msg":"sender: started","stream_id":"8gs9of7r"}
+{"time":"2025-11-06T18:28:51.825582909+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-11-06T19:15:51.940995974+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug.log

@@ -0,0 +1,21 @@
+2025-11-06 16:21:49,585 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Configure stats pid to 528220
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Loading settings from /workspace/liber/embodied_ai/lerobot-mujoco-tutorial/wandb/settings
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:setup_run_log_directory():706] Logging user logs to ckpt/smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug.log
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to ckpt/smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug-internal.log
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:init():833] calling init triggers
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'omy_pnp_language', 'root': './demo_data_language', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 5, 'n_action_steps': 5, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'optimizer_grad_clip_norm': 10, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': False, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': -1, 'pad_language_to': 'longest', 'num_expert_layers': -1, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0}, 'output_dir': 'ckpt/smolvla_omy', 'job_name': 'smolvla_omy', 'resume': False, 'seed': 42, 'num_workers': 8, 'batch_size': 16, 'steps': 20000, 'eval_freq': -1, 'log_freq': 50, 'save_checkpoint': True, 'save_freq': 5000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-10, 'grad_clip_norm': 10, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 0.0001, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'smolvla_omy', 'entity': 'ai-liber', 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
+2025-11-06 16:21:49,586 INFO    MainThread:528220 [wandb_init.py:init():881] starting backend
+2025-11-06 16:21:49,796 INFO    MainThread:528220 [wandb_init.py:init():884] sending inform_init request
+2025-11-06 16:21:49,803 INFO    MainThread:528220 [wandb_init.py:init():892] backend started and connected
+2025-11-06 16:21:49,806 INFO    MainThread:528220 [wandb_init.py:init():962] updated telemetry
+2025-11-06 16:21:49,819 INFO    MainThread:528220 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2025-11-06 16:21:51,055 INFO    MainThread:528220 [wandb_init.py:init():1033] starting run threads in backend
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_console_start():2506] atexit reg
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2025-11-06 16:21:51,410 INFO    MainThread:528220 [wandb_run.py:_redirect():2446] Redirects installed.
+2025-11-06 16:21:51,416 INFO    MainThread:528220 [wandb_init.py:init():1073] run started, returning control to user process

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/run-8gs9of7r.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:122f2a5f19ad90752dff6274a7d5884caf0566af458d12f45841bd744233e332
+size 1277952

smolvla_omy_2000_steps/checkpoints/002000/pretrained_model/config.json

@@ -0,0 +1,83 @@
+{
+    "type": "smolvla",
+    "n_obs_steps": 1,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "MEAN_STD",
+        "ACTION": "MEAN_STD"
+    },
+    "input_features": {
+        "observation.image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.wrist_image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "chunk_size": 5,
+    "n_action_steps": 5,
+    "max_state_dim": 32,
+    "max_action_dim": 32,
+    "resize_imgs_with_padding": [
+        512,
+        512
+    ],
+    "empty_cameras": 0,
+    "adapt_to_pi_aloha": false,
+    "use_delta_joint_actions_aloha": false,
+    "tokenizer_max_length": 48,
+    "num_steps": 10,
+    "use_cache": true,
+    "freeze_vision_encoder": true,
+    "train_expert_only": true,
+    "train_state_proj": true,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.95
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-10,
+    "optimizer_grad_clip_norm": 10,
+    "scheduler_warmup_steps": 1000,
+    "scheduler_decay_steps": 30000,
+    "scheduler_decay_lr": 2.5e-06,
+    "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+    "load_vlm_weights": false,
+    "add_image_special_tokens": false,
+    "attention_mode": "cross_attn",
+    "prefix_length": -1,
+    "pad_language_to": "longest",
+    "num_expert_layers": -1,
+    "num_vlm_layers": 16,
+    "self_attn_every_n_layers": 2,
+    "expert_width_multiplier": 0.75,
+    "min_period": 0.004,
+    "max_period": 4.0
+}

smolvla_omy_2000_steps/checkpoints/002000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:392c8b3292992bf112b346682fc469f9211a29aca9b37296a58fb8258502af41
+size 1197790032

smolvla_omy_2000_steps/checkpoints/002000/pretrained_model/train_config.json

@@ -0,0 +1,195 @@
+{
+    "dataset": {
+        "repo_id": "omy_pnp_language",
+        "root": "./demo_data_language",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "smolvla",
+        "n_obs_steps": 1,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "MEAN_STD",
+            "ACTION": "MEAN_STD"
+        },
+        "input_features": {
+            "observation.image": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.wrist_image": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "chunk_size": 5,
+        "n_action_steps": 5,
+        "max_state_dim": 32,
+        "max_action_dim": 32,
+        "resize_imgs_with_padding": [
+            512,
+            512
+        ],
+        "empty_cameras": 0,
+        "adapt_to_pi_aloha": false,
+        "use_delta_joint_actions_aloha": false,
+        "tokenizer_max_length": 48,
+        "num_steps": 10,
+        "use_cache": true,
+        "freeze_vision_encoder": true,
+        "train_expert_only": true,
+        "train_state_proj": true,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.95
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-10,
+        "optimizer_grad_clip_norm": 10,
+        "scheduler_warmup_steps": 1000,
+        "scheduler_decay_steps": 30000,
+        "scheduler_decay_lr": 2.5e-06,
+        "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+        "load_vlm_weights": false,
+        "add_image_special_tokens": false,
+        "attention_mode": "cross_attn",
+        "prefix_length": -1,
+        "pad_language_to": "longest",
+        "num_expert_layers": -1,
+        "num_vlm_layers": 16,
+        "self_attn_every_n_layers": 2,
+        "expert_width_multiplier": 0.75,
+        "min_period": 0.004,
+        "max_period": 4.0
+    },
+    "output_dir": "ckpt/smolvla_omy",
+    "job_name": "smolvla_omy",
+    "resume": false,
+    "seed": 42,
+    "num_workers": 8,
+    "batch_size": 16,
+    "steps": 2000,
+    "eval_freq": -1,
+    "log_freq": 50,
+    "save_checkpoint": true,
+    "save_freq": 10000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-10,
+        "grad_clip_norm": 10,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "cosine_decay_with_warmup",
+        "num_warmup_steps": 1000,
+        "num_decay_steps": 30000,
+        "peak_lr": 0.0001,
+        "decay_lr": 2.5e-06
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "smolvla_omy",
+        "entity": "ai-liber",
+        "notes": null,
+        "run_id": null,
+        "mode": null
+    }
+}

smolvla_omy_2000_steps/checkpoints/002000/training_state/optimizer_param_groups.json

@@ -0,0 +1,526 @@
+[
+    {
+        "lr": 9.893469553577303e-05,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-10,
+        "amsgrad": false,
+        "foreach": null,
+        "maximize": false,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310,
+            311,
+            312,
+            313,
+            314,
+            315,
+            316,
+            317,
+            318,
+            319,
+            320,
+            321,
+            322,
+            323,
+            324,
+            325,
+            326,
+            327,
+            328,
+            329,
+            330,
+            331,
+            332,
+            333,
+            334,
+            335,
+            336,
+            337,
+            338,
+            339,
+            340,
+            341,
+            342,
+            343,
+            344,
+            345,
+            346,
+            347,
+            348,
+            349,
+            350,
+            351,
+            352,
+            353,
+            354,
+            355,
+            356,
+            357,
+            358,
+            359,
+            360,
+            361,
+            362,
+            363,
+            364,
+            365,
+            366,
+            367,
+            368,
+            369,
+            370,
+            371,
+            372,
+            373,
+            374,
+            375,
+            376,
+            377,
+            378,
+            379,
+            380,
+            381,
+            382,
+            383,
+            384,
+            385,
+            386,
+            387,
+            388,
+            389,
+            390,
+            391,
+            392,
+            393,
+            394,
+            395,
+            396,
+            397,
+            398,
+            399,
+            400,
+            401,
+            402,
+            403,
+            404,
+            405,
+            406,
+            407,
+            408,
+            409,
+            410,
+            411,
+            412,
+            413,
+            414,
+            415,
+            416,
+            417,
+            418,
+            419,
+            420,
+            421,
+            422,
+            423,
+            424,
+            425,
+            426,
+            427,
+            428,
+            429,
+            430,
+            431,
+            432,
+            433,
+            434,
+            435,
+            436,
+            437,
+            438,
+            439,
+            440,
+            441,
+            442,
+            443,
+            444,
+            445,
+            446,
+            447,
+            448,
+            449,
+            450,
+            451,
+            452,
+            453,
+            454,
+            455,
+            456,
+            457,
+            458,
+            459,
+            460,
+            461,
+            462,
+            463,
+            464,
+            465,
+            466,
+            467,
+            468,
+            469,
+            470,
+            471,
+            472,
+            473,
+            474,
+            475,
+            476,
+            477,
+            478,
+            479,
+            480,
+            481,
+            482,
+            483,
+            484,
+            485,
+            486,
+            487,
+            488,
+            489,
+            490,
+            491,
+            492,
+            493,
+            494,
+            495,
+            496,
+            497,
+            498,
+            499,
+            500,
+            501,
+            502,
+            503,
+            504,
+            505
+        ]
+    }
+]

smolvla_omy_2000_steps/checkpoints/002000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:841e1fea0d64c05cdf1969c6274cba7a592c53080fa0dc014ba397adfde48677
+size 412659164

smolvla_omy_2000_steps/checkpoints/002000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1408160b155291236d7d16ec03ad17e4cb8484650560e2bea39128b5919b5f1
+size 15708

smolvla_omy_2000_steps/checkpoints/002000/training_state/scheduler_state.json

@@ -0,0 +1,15 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 2000,
+    "verbose": false,
+    "_step_count": 2001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        9.893469553577303e-05
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

smolvla_omy_2000_steps/checkpoints/002000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 2000
+}

smolvla_omy_2000_steps/wandb/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-11-06T15:12:53.085119529+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2025-11-06T15:12:53.65199406+08:00","level":"INFO","msg":"stream: created new stream","id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652165503+08:00","level":"INFO","msg":"handler: started","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652453415+08:00","level":"INFO","msg":"stream: started","id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652494895+08:00","level":"INFO","msg":"writer: started","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652495951+08:00","level":"INFO","msg":"sender: started","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:50:50.741281958+08:00","level":"INFO","msg":"stream: closing","id":"p7vw7943"}
+{"time":"2025-11-06T15:50:52.335321515+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-11-06T15:50:52.795892747+08:00","level":"INFO","msg":"handler: closed","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:50:52.796103474+08:00","level":"INFO","msg":"sender: closed","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:50:52.796134886+08:00","level":"INFO","msg":"stream: closed","id":"p7vw7943"}

smolvla_omy_2000_steps/wandb/debug.log

@@ -0,0 +1,23 @@
+2025-11-06 15:12:52,857 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Configure stats pid to 343428
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Loading settings from /workspace/liber/embodied_ai/lerobot-mujoco-tutorial/wandb/settings
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:setup_run_log_directory():706] Logging user logs to ckpt/smolvla_omy/wandb/run-20251106_151252-p7vw7943/logs/debug.log
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to ckpt/smolvla_omy/wandb/run-20251106_151252-p7vw7943/logs/debug-internal.log
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:init():833] calling init triggers
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'omy_pnp_language', 'root': './demo_data_language', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 5, 'n_action_steps': 5, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'optimizer_grad_clip_norm': 10, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': False, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': -1, 'pad_language_to': 'longest', 'num_expert_layers': -1, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0}, 'output_dir': 'ckpt/smolvla_omy', 'job_name': 'smolvla_omy', 'resume': False, 'seed': 42, 'num_workers': 8, 'batch_size': 16, 'steps': 2000, 'eval_freq': -1, 'log_freq': 50, 'save_checkpoint': True, 'save_freq': 10000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-10, 'grad_clip_norm': 10, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 0.0001, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'smolvla_omy', 'entity': 'ai-liber', 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:init():881] starting backend
+2025-11-06 15:12:53,069 INFO    MainThread:343428 [wandb_init.py:init():884] sending inform_init request
+2025-11-06 15:12:53,079 INFO    MainThread:343428 [wandb_init.py:init():892] backend started and connected
+2025-11-06 15:12:53,082 INFO    MainThread:343428 [wandb_init.py:init():962] updated telemetry
+2025-11-06 15:12:53,094 INFO    MainThread:343428 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2025-11-06 15:12:54,336 INFO    MainThread:343428 [wandb_init.py:init():1033] starting run threads in backend
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_console_start():2506] atexit reg
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_redirect():2446] Redirects installed.
+2025-11-06 15:12:54,689 INFO    MainThread:343428 [wandb_init.py:init():1073] run started, returning control to user process
+2025-11-06 15:50:50,741 INFO    wandb-AsyncioManager-main:343428 [service_client.py:_forward_responses():80] Reached EOF.
+2025-11-06 15:50:50,742 INFO    wandb-AsyncioManager-main:343428 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/files/config.yaml

@@ -0,0 +1,232 @@
+_wandb:
+    value:
+        cli_version: 0.22.3
+        e:
+            s9gajpxnrqtnt5al3g989j4cneoydvqt:
+                args:
+                    - --config_path
+                    - smolvla_omy.yaml
+                codePath: train_model.py
+                codePathLocal: train_model.py
+                cpu_count: 40
+                cpu_count_logical: 80
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1676411871232"
+                        used: "1480236986368"
+                executable: /opt/conda/envs/lerobot_mujoco_tutorial/bin/python
+                git:
+                    commit: b11377a0ce7a3c32cefe260c6dce959b0db2bdda
+                    remote: git@github.com:liberow/lerobot-mujoco-tutorial.git
+                gpu: Tesla V100-SXM2-32GB
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Volta
+                      cudaCores: 5120
+                      memoryTotal: "34359738368"
+                      name: Tesla V100-SXM2-32GB
+                      uuid: GPU-b29b46dc-42a3-9fcc-be17-50b8b8972960
+                    - architecture: Volta
+                      cudaCores: 5120
+                      memoryTotal: "34359738368"
+                      name: Tesla V100-SXM2-32GB
+                      uuid: GPU-368aa17c-5504-4d63-4df2-339d38a17fe3
+                    - architecture: Volta
+                      cudaCores: 5120
+                      memoryTotal: "34359738368"
+                      name: Tesla V100-SXM2-32GB
+                      uuid: GPU-acf7b3cc-f154-087a-3a06-d57e023c09be
+                    - architecture: Volta
+                      cudaCores: 5120
+                      memoryTotal: "34359738368"
+                      name: Tesla V100-SXM2-32GB
+                      uuid: GPU-86b115b3-0485-dd33-56f3-385a1140ae36
+                host: c81ab3b21da2
+                memory:
+                    total: "134767624192"
+                os: Linux-6.8.0-51-generic-x86_64-with-glibc2.35
+                program: /workspace/liber/embodied_ai/lerobot-mujoco-tutorial/train_model.py
+                python: CPython 3.10.19
+                root: ckpt/smolvla_omy
+                startedAt: "2025-11-06T07:12:52.855818Z"
+                writerId: s9gajpxnrqtnt5al3g989j4cneoydvqt
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 51
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+            "3":
+                - 13
+                - 15
+                - 16
+                - 61
+            "4": 3.10.19
+            "5": 0.22.3
+            "10":
+                - 21
+            "12": 0.22.3
+            "13": linux-x86_64
+batch_size:
+    value: 16
+dataset:
+    value:
+        episodes: null
+        image_transforms:
+            enable: false
+            max_num_transforms: 3
+            random_order: false
+            tfs:
+                brightness:
+                    kwargs:
+                        brightness:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                contrast:
+                    kwargs:
+                        contrast:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                hue:
+                    kwargs:
+                        hue:
+                            - -0.05
+                            - 0.05
+                    type: ColorJitter
+                    weight: 1
+                saturation:
+                    kwargs:
+                        saturation:
+                            - 0.5
+                            - 1.5
+                    type: ColorJitter
+                    weight: 1
+                sharpness:
+                    kwargs:
+                        sharpness:
+                            - 0.5
+                            - 1.5
+                    type: SharpnessJitter
+                    weight: 1
+        repo_id: omy_pnp_language
+        revision: null
+        root: ./demo_data_language
+        use_imagenet_stats: true
+        video_backend: torchcodec
+env:
+    value: null
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        use_async_envs: false
+eval_freq:
+    value: -1
+job_name:
+    value: smolvla_omy
+log_freq:
+    value: 50
+num_workers:
+    value: 8
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.95
+        eps: 1e-08
+        grad_clip_norm: 10
+        lr: 0.0001
+        type: adamw
+        weight_decay: 1e-10
+output_dir:
+    value: ckpt/smolvla_omy
+policy:
+    value:
+        adapt_to_pi_aloha: false
+        add_image_special_tokens: false
+        attention_mode: cross_attn
+        chunk_size: 5
+        device: cuda
+        empty_cameras: 0
+        expert_width_multiplier: 0.75
+        freeze_vision_encoder: true
+        load_vlm_weights: false
+        max_action_dim: 32
+        max_period: 4
+        max_state_dim: 32
+        min_period: 0.004
+        n_action_steps: 5
+        n_obs_steps: 1
+        normalization_mapping:
+            ACTION: MEAN_STD
+            STATE: MEAN_STD
+            VISUAL: IDENTITY
+        num_expert_layers: -1
+        num_steps: 10
+        num_vlm_layers: 16
+        optimizer_betas:
+            - 0.9
+            - 0.95
+        optimizer_eps: 1e-08
+        optimizer_grad_clip_norm: 10
+        optimizer_lr: 0.0001
+        optimizer_weight_decay: 1e-10
+        pad_language_to: longest
+        prefix_length: -1
+        resize_imgs_with_padding:
+            - 512
+            - 512
+        scheduler_decay_lr: 2.5e-06
+        scheduler_decay_steps: 30000
+        scheduler_warmup_steps: 1000
+        self_attn_every_n_layers: 2
+        tokenizer_max_length: 48
+        train_expert_only: true
+        train_state_proj: true
+        type: smolvla
+        use_amp: false
+        use_cache: true
+        use_delta_joint_actions_aloha: false
+        vlm_model_name: HuggingFaceTB/SmolVLM2-500M-Video-Instruct
+resume:
+    value: false
+save_checkpoint:
+    value: true
+save_freq:
+    value: 10000
+scheduler:
+    value:
+        decay_lr: 2.5e-06
+        num_decay_steps: 30000
+        num_warmup_steps: 1000
+        peak_lr: 0.0001
+        type: cosine_decay_with_warmup
+seed:
+    value: 42
+steps:
+    value: 2000
+use_policy_training_preset:
+    value: true
+wandb:
+    value:
+        disable_artifact: true
+        enable: true
+        entity: ai-liber
+        mode: null
+        notes: null
+        project: smolvla_omy
+        run_id: null

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/files/output.log

@@ -0,0 +1,137 @@
+Logs will be synced with wandb.
+INFO 2025-11-06 15:12:54 ndb_utils.py:96 Track this run --> https://wandb.ai/ai-liber/smolvla_omy/runs/p7vw7943
+INFO 2025-11-06 15:12:54 in_model.py:127 Creating dataset
+Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20/20 [00:00<00:00, 28397.45it/s]
+INFO 2025-11-06 15:12:56 in_model.py:138 Creating policy
+Reducing the number of VLM layers to 16 ...
+model.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 907M/907M [01:19<00:00, 11.5MB/s]
+INFO 2025-11-06 15:15:28 in_model.py:148 Creating optimizer and scheduler
+INFO 2025-11-06 15:15:28 in_model.py:160 Output dir: ckpt/smolvla_omy
+INFO 2025-11-06 15:15:28 in_model.py:163 cfg.steps=2000 (2K)
+INFO 2025-11-06 15:15:28 in_model.py:164 dataset.num_frames=2621 (3K)
+INFO 2025-11-06 15:15:28 in_model.py:165 dataset.num_episodes=20
+INFO 2025-11-06 15:15:28 in_model.py:166 num_learnable_params=99880992 (100M)
+INFO 2025-11-06 15:15:28 in_model.py:167 num_total_params=450046216 (450M)
+INFO 2025-11-06 15:15:28 in_model.py:206 Start offline training on a fixed dataset
+INFO 2025-11-06 15:16:19 in_model.py:236 step:50 smpl:800 ep:6 epch:0.31 loss:0.306 grdn:2.807 lr:2.6e-06 updt_s:1.004 data_s:0.025
+WARNING 2025-11-06 15:16:19 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:16:19 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:17:09 in_model.py:236 step:100 smpl:2K ep:12 epch:0.61 loss:0.223 grdn:1.959 lr:7.6e-06 updt_s:0.986 data_s:0.000
+WARNING 2025-11-06 15:17:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:17:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:17:59 in_model.py:236 step:150 smpl:2K ep:18 epch:0.92 loss:0.166 grdn:1.444 lr:1.3e-05 updt_s:1.000 data_s:0.000
+WARNING 2025-11-06 15:17:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:17:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:18:53 in_model.py:236 step:200 smpl:3K ep:24 epch:1.22 loss:0.112 grdn:1.042 lr:1.8e-05 updt_s:1.040 data_s:0.030
+WARNING 2025-11-06 15:18:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:18:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:19:45 in_model.py:236 step:250 smpl:4K ep:31 epch:1.53 loss:0.091 grdn:0.947 lr:2.3e-05 updt_s:1.050 data_s:0.000
+WARNING 2025-11-06 15:19:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:19:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:20:38 in_model.py:236 step:300 smpl:5K ep:37 epch:1.83 loss:0.079 grdn:0.972 lr:2.8e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 15:20:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:20:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:21:32 in_model.py:236 step:350 smpl:6K ep:43 epch:2.14 loss:0.068 grdn:0.944 lr:3.3e-05 updt_s:1.045 data_s:0.022
+WARNING 2025-11-06 15:21:32 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:21:32 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:22:25 in_model.py:236 step:400 smpl:6K ep:49 epch:2.44 loss:0.062 grdn:0.921 lr:3.8e-05 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 15:22:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:22:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:23:17 in_model.py:236 step:450 smpl:7K ep:55 epch:2.75 loss:0.062 grdn:0.979 lr:4.3e-05 updt_s:1.051 data_s:0.000
+WARNING 2025-11-06 15:23:17 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:23:17 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:24:11 in_model.py:236 step:500 smpl:8K ep:61 epch:3.05 loss:0.055 grdn:0.987 lr:4.8e-05 updt_s:1.044 data_s:0.022
+WARNING 2025-11-06 15:24:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:24:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:25:03 in_model.py:236 step:550 smpl:9K ep:67 epch:3.36 loss:0.046 grdn:0.843 lr:5.3e-05 updt_s:1.049 data_s:0.000
+WARNING 2025-11-06 15:25:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:25:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:25:56 in_model.py:236 step:600 smpl:10K ep:73 epch:3.66 loss:0.050 grdn:0.955 lr:5.8e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 15:25:56 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:25:56 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:26:49 in_model.py:236 step:650 smpl:10K ep:79 epch:3.97 loss:0.050 grdn:1.030 lr:6.3e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 15:26:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:26:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:27:43 in_model.py:236 step:700 smpl:11K ep:85 epch:4.27 loss:0.044 grdn:0.878 lr:6.8e-05 updt_s:1.047 data_s:0.020
+WARNING 2025-11-06 15:27:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:27:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:28:36 in_model.py:236 step:750 smpl:12K ep:92 epch:4.58 loss:0.048 grdn:0.911 lr:7.3e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 15:28:36 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:28:36 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:29:29 in_model.py:236 step:800 smpl:13K ep:98 epch:4.88 loss:0.050 grdn:0.946 lr:7.8e-05 updt_s:1.055 data_s:0.000
+WARNING 2025-11-06 15:29:29 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:29:29 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:30:23 in_model.py:236 step:850 smpl:14K ep:104 epch:5.19 loss:0.045 grdn:0.904 lr:8.3e-05 updt_s:1.047 data_s:0.021
+WARNING 2025-11-06 15:30:23 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:30:23 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:31:15 in_model.py:236 step:900 smpl:14K ep:110 epch:5.49 loss:0.056 grdn:1.043 lr:8.8e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 15:31:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:31:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:32:09 in_model.py:236 step:950 smpl:15K ep:116 epch:5.80 loss:0.056 grdn:1.051 lr:9.3e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 15:32:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:32:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:33:02 in_model.py:236 step:1K smpl:16K ep:122 epch:6.10 loss:0.051 grdn:0.937 lr:9.8e-05 updt_s:1.047 data_s:0.021
+WARNING 2025-11-06 15:33:02 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:33:02 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:33:55 in_model.py:236 step:1K smpl:17K ep:128 epch:6.41 loss:0.047 grdn:0.883 lr:1.0e-04 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 15:33:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:33:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:34:48 in_model.py:236 step:1K smpl:18K ep:134 epch:6.71 loss:0.056 grdn:0.951 lr:1.0e-04 updt_s:1.057 data_s:0.000
+WARNING 2025-11-06 15:34:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:34:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:35:42 in_model.py:236 step:1K smpl:18K ep:140 epch:7.02 loss:0.055 grdn:0.955 lr:1.0e-04 updt_s:1.051 data_s:0.020
+WARNING 2025-11-06 15:35:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:35:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:36:34 in_model.py:236 step:1K smpl:19K ep:147 epch:7.33 loss:0.049 grdn:0.953 lr:1.0e-04 updt_s:1.050 data_s:0.000
+WARNING 2025-11-06 15:36:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:36:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:37:27 in_model.py:236 step:1K smpl:20K ep:153 epch:7.63 loss:0.045 grdn:0.868 lr:1.0e-04 updt_s:1.052 data_s:0.000
+WARNING 2025-11-06 15:37:27 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:37:27 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:38:20 in_model.py:236 step:1K smpl:21K ep:159 epch:7.94 loss:0.053 grdn:0.922 lr:1.0e-04 updt_s:1.058 data_s:0.000
+WARNING 2025-11-06 15:38:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:38:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:39:14 in_model.py:236 step:1K smpl:22K ep:165 epch:8.24 loss:0.054 grdn:0.953 lr:1.0e-04 updt_s:1.048 data_s:0.021
+WARNING 2025-11-06 15:39:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:39:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:40:07 in_model.py:236 step:1K smpl:22K ep:171 epch:8.55 loss:0.044 grdn:0.778 lr:9.9e-05 updt_s:1.059 data_s:0.000
+WARNING 2025-11-06 15:40:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:40:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:41:00 in_model.py:236 step:1K smpl:23K ep:177 epch:8.85 loss:0.044 grdn:0.855 lr:9.9e-05 updt_s:1.054 data_s:0.000
+WARNING 2025-11-06 15:41:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:41:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:41:54 in_model.py:236 step:2K smpl:24K ep:183 epch:9.16 loss:0.045 grdn:0.835 lr:9.9e-05 updt_s:1.055 data_s:0.021
+WARNING 2025-11-06 15:41:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:41:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:42:47 in_model.py:236 step:2K smpl:25K ep:189 epch:9.46 loss:0.043 grdn:0.803 lr:9.9e-05 updt_s:1.063 data_s:0.000
+WARNING 2025-11-06 15:42:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:42:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:43:41 in_model.py:236 step:2K smpl:26K ep:195 epch:9.77 loss:0.047 grdn:0.870 lr:9.9e-05 updt_s:1.065 data_s:0.000
+WARNING 2025-11-06 15:43:41 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:43:41 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:44:35 in_model.py:236 step:2K smpl:26K ep:201 epch:10.07 loss:0.042 grdn:0.784 lr:9.9e-05 updt_s:1.052 data_s:0.022
+WARNING 2025-11-06 15:44:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:44:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:45:27 in_model.py:236 step:2K smpl:27K ep:208 epch:10.38 loss:0.043 grdn:0.775 lr:9.9e-05 updt_s:1.050 data_s:0.000
+WARNING 2025-11-06 15:45:27 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:45:27 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:46:20 in_model.py:236 step:2K smpl:28K ep:214 epch:10.68 loss:0.042 grdn:0.799 lr:9.9e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 15:46:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:46:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:47:13 in_model.py:236 step:2K smpl:29K ep:220 epch:10.99 loss:0.038 grdn:0.714 lr:9.9e-05 updt_s:1.053 data_s:0.000
+WARNING 2025-11-06 15:47:13 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:47:13 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:48:07 in_model.py:236 step:2K smpl:30K ep:226 epch:11.29 loss:0.040 grdn:0.753 lr:9.9e-05 updt_s:1.047 data_s:0.021
+WARNING 2025-11-06 15:48:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:48:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:49:00 in_model.py:236 step:2K smpl:30K ep:232 epch:11.60 loss:0.038 grdn:0.699 lr:9.9e-05 updt_s:1.059 data_s:0.000
+WARNING 2025-11-06 15:49:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:49:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:49:53 in_model.py:236 step:2K smpl:31K ep:238 epch:11.90 loss:0.038 grdn:0.660 lr:9.9e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 15:49:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:49:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:50:47 in_model.py:236 step:2K smpl:32K ep:244 epch:12.21 loss:0.034 grdn:0.633 lr:9.9e-05 updt_s:1.048 data_s:0.021
+WARNING 2025-11-06 15:50:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 15:50:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 15:50:47 in_model.py:245 Checkpoint policy after step 2000
+INFO 2025-11-06 15:50:50 in_model.py:287 End of training

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/files/requirements.txt

@@ -0,0 +1,172 @@
+pyparsing==3.2.5
+nvidia-cuda-cupti-cu12==12.4.127
+pip==25.2
+setuptools==80.9.0
+gitdb==4.0.12
+networkx==3.4.2
+propcache==0.4.1
+aiohappyeyeballs==2.6.1
+aiosignal==1.4.0
+Werkzeug==3.1.3
+nvidia-nvtx-cu12==12.4.127
+typing-inspect==0.9.0
+urllib3==2.5.0
+imageio-ffmpeg==0.6.0
+num2words==0.5.14
+PySocks==1.7.1
+datasets==3.4.1
+PyScreeze==1.0.1
+tokenizers==0.21.4
+yarl==1.22.0
+MouseInfo==0.1.3
+mypy_extensions==1.1.0
+pyzmq==27.1.0
+beautifulsoup4==4.14.2
+importlib_metadata==8.7.0
+evdev==1.9.2
+sentry-sdk==2.43.0
+wcwidth==0.2.14
+typing_extensions==4.15.0
+certifi==2025.10.5
+importlib_resources==6.5.2
+hf-xet==1.2.0
+frozenlist==1.8.0
+annotated-types==0.7.0
+PyGetWindow==0.0.9
+draccus==0.10.0
+rerun-sdk==0.26.2
+cmake==4.1.2
+PyAutoGUI==0.9.54
+gdown==5.2.0
+pyarrow==22.0.0
+nvidia-cusparse-cu12==12.3.1.170
+torch==2.6.0
+cycler==0.12.1
+av==16.0.1
+pytz==2025.2
+tzdata==2025.2
+zipp==3.23.0
+prompt_toolkit==3.0.52
+itsdangerous==2.2.0
+lerobot==0.1.0
+smmap==5.0.2
+pyserial==3.5
+charset-normalizer==3.4.4
+numpy==2.2.6
+psutil==7.1.3
+zarr==2.18.3
+matplotlib==3.10.7
+llvmlite==0.45.1
+torchaudio==2.6.0
+huggingface-hub==0.36.0
+MarkupSafe==3.0.3
+packaging==25.0
+PyYAML==6.0.3
+Flask==3.1.2
+hf_transfer==0.1.9
+pydantic_core==2.41.4
+pandas==2.3.3
+mergedeep==1.3.4
+h5py==3.15.1
+transformers==4.50.3
+torchcodec==0.8.1
+termcolor==3.2.0
+toml==0.10.2
+imageio==2.37.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+asciitree==0.3.3
+antlr4-python3-runtime==4.9.3
+jsonlines==4.0.0
+scipy==1.15.3
+pyperclip==1.11.0
+pillow==12.0.0
+click==8.3.0
+torchvision==0.21.0
+cffi==2.0.0
+etils==1.13.0
+Farama-Notifications==0.0.4
+nvidia-cufft-cu12==11.2.1.3
+kiwisolver==1.4.9
+triton==3.2.0
+fonttools==4.60.1
+pycparser==2.23
+absl-py==2.3.1
+requests==2.32.5
+six==1.17.0
+nvidia-nccl-cu12==2.21.5
+pytweening==1.2.0
+docopt==0.6.2
+contourpy==1.3.2
+nvidia-cusolver-cu12==11.6.1.9
+pynput==1.8.1
+Jinja2==3.1.6
+einops==0.8.1
+numcodecs==0.13.1
+async-timeout==5.0.1
+blinker==1.9.0
+pydantic==2.12.3
+xxhash==3.6.0
+cloudpickle==3.1.2
+diffusers==0.35.2
+PyOpenGL==3.1.10
+glfw==2.10.0
+soupsieve==2.8
+GitPython==3.1.45
+fsspec==2024.12.0
+aiohttp==3.13.2
+opencv-python-headless==4.12.0.88
+gymnasium==0.29.1
+python3-xlib==0.15
+mpmath==1.3.0
+nvidia-curand-cu12==10.3.5.147
+wandb==0.22.3
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-runtime-cu12==12.4.127
+python-xlib==0.33
+multiprocess==0.70.16
+pfzy==0.3.4
+orderly-set==5.5.0
+nvidia-nvjitlink-cu12==12.4.127
+accelerate==1.11.0
+regex==2025.11.3
+omegaconf==2.3.0
+safetensors==0.5.3
+filelock==3.20.0
+numba==0.62.1
+nvidia-cusparselt-cu12==0.6.2
+dill==0.3.8
+pyyaml-include==1.4.1
+platformdirs==4.5.0
+multidict==6.7.0
+protobuf==6.33.0
+attrs==25.4.0
+inquirerpy==0.3.4
+PyMsgBox==2.0.1
+pymunk==6.11.1
+nvidia-cudnn-cu12==9.1.0.70
+typing-inspection==0.4.2
+fasteners==0.20
+PyRect==0.2.0
+idna==3.11
+deepdiff==8.6.1
+wheel==0.45.1
+sympy==1.13.1
+python-dateutil==2.9.0.post0
+mujoco==3.1.6
+tqdm==4.67.1
+backports.tarfile==1.2.0
+typeguard==4.3.0
+zipp==3.19.2
+inflect==7.3.1
+more-itertools==10.3.0
+tomli==2.0.1
+jaraco.collections==5.1.0
+jaraco.functools==4.0.1
+typing_extensions==4.12.2
+importlib_metadata==8.0.0
+jaraco.text==3.12.1
+platformdirs==4.2.2
+jaraco.context==5.3.0
+wheel==0.45.1
+packaging==24.2
+autocommand==2.2.2

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/files/wandb-metadata.json

@@ -0,0 +1,64 @@
+{
+  "os":  "Linux-6.8.0-51-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.19",
+  "startedAt":  "2025-11-06T07:12:52.855818Z",
+  "args":  [
+    "--config_path",
+    "smolvla_omy.yaml"
+  ],
+  "program":  "/workspace/liber/embodied_ai/lerobot-mujoco-tutorial/train_model.py",
+  "codePath":  "train_model.py",
+  "codePathLocal":  "train_model.py",
+  "git":  {
+    "remote":  "git@github.com:liberow/lerobot-mujoco-tutorial.git",
+    "commit":  "b11377a0ce7a3c32cefe260c6dce959b0db2bdda"
+  },
+  "root":  "ckpt/smolvla_omy",
+  "host":  "c81ab3b21da2",
+  "executable":  "/opt/conda/envs/lerobot_mujoco_tutorial/bin/python",
+  "cpu_count":  40,
+  "cpu_count_logical":  80,
+  "gpu":  "Tesla V100-SXM2-32GB",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "1676411871232",
+      "used":  "1480236986368"
+    }
+  },
+  "memory":  {
+    "total":  "134767624192"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-b29b46dc-42a3-9fcc-be17-50b8b8972960"
+    },
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-368aa17c-5504-4d63-4df2-339d38a17fe3"
+    },
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-acf7b3cc-f154-087a-3a06-d57e023c09be"
+    },
+    {
+      "name":  "Tesla V100-SXM2-32GB",
+      "memoryTotal":  "34359738368",
+      "cudaCores":  5120,
+      "architecture":  "Volta",
+      "uuid":  "GPU-86b115b3-0485-dd33-56f3-385a1140ae36"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "s9gajpxnrqtnt5al3g989j4cneoydvqt"
+}

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_timestamp":1.7624154471798701e+09,"train/epochs":12.209080503624572,"_wandb":{"runtime":2276},"train/lr":9.896048857046275e-05,"train/samples":32000,"_runtime":2276.402947135,"train/update_s":1.0481160825677216,"_step":2000,"train/steps":2000,"train/grad_norm":0.6333963936567306,"train/episodes":244.1816100724914,"train/loss":0.025371486321091652,"train/dataloading_s":0.02116153893992305}

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-11-06T15:12:52.895741661+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp0l06am8l/port-343428.txt","pid":343428,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-11-06T15:12:52.897386616+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":343428}
+{"time":"2025-11-06T15:12:52.897390339+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-343428-344296-4240599004/socket","Net":"unix"}}
+{"time":"2025-11-06T15:12:53.068133217+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-11-06T15:12:53.084869085+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"p7vw7943","id":"1(@)"}
+{"time":"2025-11-06T15:12:53.652476859+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"p7vw7943","id":"1(@)"}
+{"time":"2025-11-06T15:50:50.741198323+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-11-06T15:50:50.741293234+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-11-06T15:50:50.741322423+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-11-06T15:50:50.741481914+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-11-06T15:50:50.741595273+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-343428-344296-4240599004/socket","Net":"unix"}}
+{"time":"2025-11-06T15:50:52.796486185+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-11-06T15:50:52.796535792+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-11-06T15:50:52.796557312+08:00","level":"INFO","msg":"server is closed"}

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-11-06T15:12:53.085119529+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2025-11-06T15:12:53.65199406+08:00","level":"INFO","msg":"stream: created new stream","id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652165503+08:00","level":"INFO","msg":"handler: started","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652453415+08:00","level":"INFO","msg":"stream: started","id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652494895+08:00","level":"INFO","msg":"writer: started","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:12:53.652495951+08:00","level":"INFO","msg":"sender: started","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:50:50.741281958+08:00","level":"INFO","msg":"stream: closing","id":"p7vw7943"}
+{"time":"2025-11-06T15:50:52.335321515+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-11-06T15:50:52.795892747+08:00","level":"INFO","msg":"handler: closed","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:50:52.796103474+08:00","level":"INFO","msg":"sender: closed","stream_id":"p7vw7943"}
+{"time":"2025-11-06T15:50:52.796134886+08:00","level":"INFO","msg":"stream: closed","id":"p7vw7943"}

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/logs/debug.log

@@ -0,0 +1,23 @@
+2025-11-06 15:12:52,857 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Configure stats pid to 343428
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Loading settings from /workspace/liber/embodied_ai/lerobot-mujoco-tutorial/wandb/settings
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:setup_run_log_directory():706] Logging user logs to ckpt/smolvla_omy/wandb/run-20251106_151252-p7vw7943/logs/debug.log
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to ckpt/smolvla_omy/wandb/run-20251106_151252-p7vw7943/logs/debug-internal.log
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:init():833] calling init triggers
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'omy_pnp_language', 'root': './demo_data_language', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 5, 'n_action_steps': 5, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-10, 'optimizer_grad_clip_norm': 10, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': False, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': -1, 'pad_language_to': 'longest', 'num_expert_layers': -1, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0}, 'output_dir': 'ckpt/smolvla_omy', 'job_name': 'smolvla_omy', 'resume': False, 'seed': 42, 'num_workers': 8, 'batch_size': 16, 'steps': 2000, 'eval_freq': -1, 'log_freq': 50, 'save_checkpoint': True, 'save_freq': 10000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-10, 'grad_clip_norm': 10, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 0.0001, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'smolvla_omy', 'entity': 'ai-liber', 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
+2025-11-06 15:12:52,858 INFO    MainThread:343428 [wandb_init.py:init():881] starting backend
+2025-11-06 15:12:53,069 INFO    MainThread:343428 [wandb_init.py:init():884] sending inform_init request
+2025-11-06 15:12:53,079 INFO    MainThread:343428 [wandb_init.py:init():892] backend started and connected
+2025-11-06 15:12:53,082 INFO    MainThread:343428 [wandb_init.py:init():962] updated telemetry
+2025-11-06 15:12:53,094 INFO    MainThread:343428 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2025-11-06 15:12:54,336 INFO    MainThread:343428 [wandb_init.py:init():1033] starting run threads in backend
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_console_start():2506] atexit reg
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2025-11-06 15:12:54,684 INFO    MainThread:343428 [wandb_run.py:_redirect():2446] Redirects installed.
+2025-11-06 15:12:54,689 INFO    MainThread:343428 [wandb_init.py:init():1073] run started, returning control to user process
+2025-11-06 15:50:50,741 INFO    wandb-AsyncioManager-main:343428 [service_client.py:_forward_responses():80] Reached EOF.
+2025-11-06 15:50:50,742 INFO    wandb-AsyncioManager-main:343428 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

smolvla_omy_2000_steps/wandb/run-20251106_151252-p7vw7943/run-p7vw7943.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f8032f5860ee1d4d5c37feb52286d1469d23d14b6bb370b766d431d10a92f6
+size 418963