Upload folder using huggingface_hub

smolvla_omy/checkpoints/015000/pretrained_model/config.json

@@ -0,0 +1,83 @@
+{
+    "type": "smolvla",
+    "n_obs_steps": 1,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "MEAN_STD",
+        "ACTION": "MEAN_STD"
+    },
+    "input_features": {
+        "observation.image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.wrist_image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "chunk_size": 5,
+    "n_action_steps": 5,
+    "max_state_dim": 32,
+    "max_action_dim": 32,
+    "resize_imgs_with_padding": [
+        512,
+        512
+    ],
+    "empty_cameras": 0,
+    "adapt_to_pi_aloha": false,
+    "use_delta_joint_actions_aloha": false,
+    "tokenizer_max_length": 48,
+    "num_steps": 10,
+    "use_cache": true,
+    "freeze_vision_encoder": true,
+    "train_expert_only": true,
+    "train_state_proj": true,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.95
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-10,
+    "optimizer_grad_clip_norm": 10,
+    "scheduler_warmup_steps": 1000,
+    "scheduler_decay_steps": 30000,
+    "scheduler_decay_lr": 2.5e-06,
+    "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+    "load_vlm_weights": false,
+    "add_image_special_tokens": false,
+    "attention_mode": "cross_attn",
+    "prefix_length": -1,
+    "pad_language_to": "longest",
+    "num_expert_layers": -1,
+    "num_vlm_layers": 16,
+    "self_attn_every_n_layers": 2,
+    "expert_width_multiplier": 0.75,
+    "min_period": 0.004,
+    "max_period": 4.0
+}

smolvla_omy/checkpoints/015000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe5be9c29d5d6d9ed380929e4cbb01436d249589f9368ceedc6f17c8f09a0e3e
+size 1197790032

smolvla_omy/checkpoints/015000/pretrained_model/train_config.json

@@ -0,0 +1,195 @@
+{
+    "dataset": {
+        "repo_id": "omy_pnp_language",
+        "root": "./demo_data_language",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "smolvla",
+        "n_obs_steps": 1,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "MEAN_STD",
+            "ACTION": "MEAN_STD"
+        },
+        "input_features": {
+            "observation.image": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.wrist_image": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    256,
+                    256
+                ]
+            },
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "chunk_size": 5,
+        "n_action_steps": 5,
+        "max_state_dim": 32,
+        "max_action_dim": 32,
+        "resize_imgs_with_padding": [
+            512,
+            512
+        ],
+        "empty_cameras": 0,
+        "adapt_to_pi_aloha": false,
+        "use_delta_joint_actions_aloha": false,
+        "tokenizer_max_length": 48,
+        "num_steps": 10,
+        "use_cache": true,
+        "freeze_vision_encoder": true,
+        "train_expert_only": true,
+        "train_state_proj": true,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.95
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-10,
+        "optimizer_grad_clip_norm": 10,
+        "scheduler_warmup_steps": 1000,
+        "scheduler_decay_steps": 30000,
+        "scheduler_decay_lr": 2.5e-06,
+        "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+        "load_vlm_weights": false,
+        "add_image_special_tokens": false,
+        "attention_mode": "cross_attn",
+        "prefix_length": -1,
+        "pad_language_to": "longest",
+        "num_expert_layers": -1,
+        "num_vlm_layers": 16,
+        "self_attn_every_n_layers": 2,
+        "expert_width_multiplier": 0.75,
+        "min_period": 0.004,
+        "max_period": 4.0
+    },
+    "output_dir": "ckpt/smolvla_omy",
+    "job_name": "smolvla_omy",
+    "resume": false,
+    "seed": 42,
+    "num_workers": 8,
+    "batch_size": 16,
+    "steps": 20000,
+    "eval_freq": -1,
+    "log_freq": 50,
+    "save_checkpoint": true,
+    "save_freq": 5000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-10,
+        "grad_clip_norm": 10,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "cosine_decay_with_warmup",
+        "num_warmup_steps": 1000,
+        "num_decay_steps": 30000,
+        "peak_lr": 0.0001,
+        "decay_lr": 2.5e-06
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "smolvla_omy",
+        "entity": "ai-liber",
+        "notes": null,
+        "run_id": null,
+        "mode": null
+    }
+}

smolvla_omy/checkpoints/015000/training_state/optimizer_param_groups.json

@@ -0,0 +1,526 @@
+[
+    {
+        "lr": 5.125e-05,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-10,
+        "amsgrad": false,
+        "foreach": null,
+        "maximize": false,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310,
+            311,
+            312,
+            313,
+            314,
+            315,
+            316,
+            317,
+            318,
+            319,
+            320,
+            321,
+            322,
+            323,
+            324,
+            325,
+            326,
+            327,
+            328,
+            329,
+            330,
+            331,
+            332,
+            333,
+            334,
+            335,
+            336,
+            337,
+            338,
+            339,
+            340,
+            341,
+            342,
+            343,
+            344,
+            345,
+            346,
+            347,
+            348,
+            349,
+            350,
+            351,
+            352,
+            353,
+            354,
+            355,
+            356,
+            357,
+            358,
+            359,
+            360,
+            361,
+            362,
+            363,
+            364,
+            365,
+            366,
+            367,
+            368,
+            369,
+            370,
+            371,
+            372,
+            373,
+            374,
+            375,
+            376,
+            377,
+            378,
+            379,
+            380,
+            381,
+            382,
+            383,
+            384,
+            385,
+            386,
+            387,
+            388,
+            389,
+            390,
+            391,
+            392,
+            393,
+            394,
+            395,
+            396,
+            397,
+            398,
+            399,
+            400,
+            401,
+            402,
+            403,
+            404,
+            405,
+            406,
+            407,
+            408,
+            409,
+            410,
+            411,
+            412,
+            413,
+            414,
+            415,
+            416,
+            417,
+            418,
+            419,
+            420,
+            421,
+            422,
+            423,
+            424,
+            425,
+            426,
+            427,
+            428,
+            429,
+            430,
+            431,
+            432,
+            433,
+            434,
+            435,
+            436,
+            437,
+            438,
+            439,
+            440,
+            441,
+            442,
+            443,
+            444,
+            445,
+            446,
+            447,
+            448,
+            449,
+            450,
+            451,
+            452,
+            453,
+            454,
+            455,
+            456,
+            457,
+            458,
+            459,
+            460,
+            461,
+            462,
+            463,
+            464,
+            465,
+            466,
+            467,
+            468,
+            469,
+            470,
+            471,
+            472,
+            473,
+            474,
+            475,
+            476,
+            477,
+            478,
+            479,
+            480,
+            481,
+            482,
+            483,
+            484,
+            485,
+            486,
+            487,
+            488,
+            489,
+            490,
+            491,
+            492,
+            493,
+            494,
+            495,
+            496,
+            497,
+            498,
+            499,
+            500,
+            501,
+            502,
+            503,
+            504,
+            505
+        ]
+    }
+]

smolvla_omy/checkpoints/015000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a1e0078b6ffe1de6e1be8cb9ab5ca3068fbb9ad6b4febc43eb6eee135c3f280
+size 412659164

smolvla_omy/checkpoints/015000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ac44f7d5de34721a9025daf3961ad13006b45aea7755c2ef184c82f5c10252
+size 15708

smolvla_omy/checkpoints/015000/training_state/scheduler_state.json

@@ -0,0 +1,15 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 15000,
+    "verbose": false,
+    "_step_count": 15001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        5.125e-05
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

smolvla_omy/checkpoints/015000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 15000
+}

smolvla_omy/wandb/debug-internal.log

@@ -6,3 +6,4 @@
 {"time":"2025-11-06T16:21:50.381975054+08:00","level":"INFO","msg":"sender: started","stream_id":"8gs9of7r"}
 {"time":"2025-11-06T18:28:51.825582909+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
 {"time":"2025-11-06T19:15:51.940995974+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-11-06T20:24:22.8723501+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/files/output.log

@@ -656,3 +656,328 @@ WARNING 2025-11-06 19:30:38 db_utils.py:117 WandB logging of key "losses_after_r
 INFO 2025-11-06 19:31:31 in_model.py:236 step:11K smpl:171K ep:1K epch:65.32 loss:0.015 grdn:0.365 lr:7.3e-05 updt_s:1.021 data_s:0.020
 WARNING 2025-11-06 19:31:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
 WARNING 2025-11-06 19:31:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:32:22 in_model.py:236 step:11K smpl:172K ep:1K epch:65.62 loss:0.013 grdn:0.334 lr:7.2e-05 updt_s:1.025 data_s:0.000
+WARNING 2025-11-06 19:32:22 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:32:22 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:33:14 in_model.py:236 step:11K smpl:173K ep:1K epch:65.93 loss:0.016 grdn:0.397 lr:7.2e-05 updt_s:1.029 data_s:0.000
+WARNING 2025-11-06 19:33:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:33:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:34:06 in_model.py:236 step:11K smpl:174K ep:1K epch:66.23 loss:0.013 grdn:0.369 lr:7.2e-05 updt_s:1.022 data_s:0.021
+WARNING 2025-11-06 19:34:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:34:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:34:57 in_model.py:236 step:11K smpl:174K ep:1K epch:66.54 loss:0.014 grdn:0.369 lr:7.2e-05 updt_s:1.030 data_s:0.000
+WARNING 2025-11-06 19:34:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:34:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:35:49 in_model.py:236 step:11K smpl:175K ep:1K epch:66.84 loss:0.014 grdn:0.344 lr:7.1e-05 updt_s:1.028 data_s:0.000
+WARNING 2025-11-06 19:35:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:35:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:36:41 in_model.py:236 step:11K smpl:176K ep:1K epch:67.15 loss:0.013 grdn:0.351 lr:7.1e-05 updt_s:1.023 data_s:0.021
+WARNING 2025-11-06 19:36:41 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:36:41 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:37:33 in_model.py:236 step:11K smpl:177K ep:1K epch:67.46 loss:0.013 grdn:0.361 lr:7.1e-05 updt_s:1.028 data_s:0.000
+WARNING 2025-11-06 19:37:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:37:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:38:24 in_model.py:236 step:11K smpl:178K ep:1K epch:67.76 loss:0.014 grdn:0.346 lr:7.1e-05 updt_s:1.027 data_s:0.000
+WARNING 2025-11-06 19:38:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:38:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:39:16 in_model.py:236 step:11K smpl:178K ep:1K epch:68.07 loss:0.012 grdn:0.322 lr:7.0e-05 updt_s:1.029 data_s:0.000
+WARNING 2025-11-06 19:39:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:39:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:40:08 in_model.py:236 step:11K smpl:179K ep:1K epch:68.37 loss:0.013 grdn:0.345 lr:7.0e-05 updt_s:1.017 data_s:0.021
+WARNING 2025-11-06 19:40:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:40:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:40:59 in_model.py:236 step:11K smpl:180K ep:1K epch:68.68 loss:0.012 grdn:0.316 lr:7.0e-05 updt_s:1.024 data_s:0.000
+WARNING 2025-11-06 19:40:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:40:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:41:51 in_model.py:236 step:11K smpl:181K ep:1K epch:68.98 loss:0.015 grdn:0.373 lr:7.0e-05 updt_s:1.029 data_s:0.000
+WARNING 2025-11-06 19:41:51 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:41:51 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:42:43 in_model.py:236 step:11K smpl:182K ep:1K epch:69.29 loss:0.013 grdn:0.321 lr:7.0e-05 updt_s:1.020 data_s:0.021
+WARNING 2025-11-06 19:42:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:42:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:43:34 in_model.py:236 step:11K smpl:182K ep:1K epch:69.59 loss:0.012 grdn:0.315 lr:6.9e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 19:43:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:43:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:44:26 in_model.py:236 step:11K smpl:183K ep:1K epch:69.90 loss:0.015 grdn:0.381 lr:6.9e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 19:44:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:44:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:45:18 in_model.py:236 step:12K smpl:184K ep:1K epch:70.20 loss:0.013 grdn:0.324 lr:6.9e-05 updt_s:1.015 data_s:0.022
+WARNING 2025-11-06 19:45:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:45:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:46:09 in_model.py:236 step:12K smpl:185K ep:1K epch:70.51 loss:0.013 grdn:0.342 lr:6.9e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 19:46:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:46:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:47:00 in_model.py:236 step:12K smpl:186K ep:1K epch:70.81 loss:0.013 grdn:0.334 lr:6.8e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 19:47:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:47:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:47:52 in_model.py:236 step:12K smpl:186K ep:1K epch:71.12 loss:0.015 grdn:0.360 lr:6.8e-05 updt_s:1.017 data_s:0.021
+WARNING 2025-11-06 19:47:52 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:47:52 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:48:44 in_model.py:236 step:12K smpl:187K ep:1K epch:71.42 loss:0.014 grdn:0.356 lr:6.8e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 19:48:44 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:48:44 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:49:35 in_model.py:236 step:12K smpl:188K ep:1K epch:71.73 loss:0.015 grdn:0.356 lr:6.8e-05 updt_s:1.024 data_s:0.000
+WARNING 2025-11-06 19:49:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:49:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:50:26 in_model.py:236 step:12K smpl:189K ep:1K epch:72.03 loss:0.010 grdn:0.269 lr:6.7e-05 updt_s:1.024 data_s:0.000
+WARNING 2025-11-06 19:50:26 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:50:26 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:51:18 in_model.py:236 step:12K smpl:190K ep:1K epch:72.34 loss:0.012 grdn:0.331 lr:6.7e-05 updt_s:1.018 data_s:0.021
+WARNING 2025-11-06 19:51:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:51:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:52:10 in_model.py:236 step:12K smpl:190K ep:1K epch:72.64 loss:0.014 grdn:0.361 lr:6.7e-05 updt_s:1.025 data_s:0.000
+WARNING 2025-11-06 19:52:10 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:52:10 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:53:01 in_model.py:236 step:12K smpl:191K ep:1K epch:72.95 loss:0.013 grdn:0.378 lr:6.7e-05 updt_s:1.026 data_s:0.000
+WARNING 2025-11-06 19:53:01 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:53:01 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:53:53 in_model.py:236 step:12K smpl:192K ep:1K epch:73.25 loss:0.014 grdn:0.337 lr:6.6e-05 updt_s:1.019 data_s:0.022
+WARNING 2025-11-06 19:53:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:53:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:54:45 in_model.py:236 step:12K smpl:193K ep:1K epch:73.56 loss:0.015 grdn:0.384 lr:6.6e-05 updt_s:1.024 data_s:0.000
+WARNING 2025-11-06 19:54:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:54:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:55:36 in_model.py:236 step:12K smpl:194K ep:1K epch:73.86 loss:0.012 grdn:0.326 lr:6.6e-05 updt_s:1.025 data_s:0.000
+WARNING 2025-11-06 19:55:36 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:55:36 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:56:28 in_model.py:236 step:12K smpl:194K ep:1K epch:74.17 loss:0.013 grdn:0.328 lr:6.6e-05 updt_s:1.017 data_s:0.021
+WARNING 2025-11-06 19:56:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:56:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:57:20 in_model.py:236 step:12K smpl:195K ep:1K epch:74.48 loss:0.011 grdn:0.316 lr:6.5e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 19:57:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:57:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:58:11 in_model.py:236 step:12K smpl:196K ep:1K epch:74.78 loss:0.017 grdn:0.380 lr:6.5e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 19:58:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:58:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:59:02 in_model.py:236 step:12K smpl:197K ep:2K epch:75.09 loss:0.012 grdn:0.313 lr:6.5e-05 updt_s:1.019 data_s:0.000
+WARNING 2025-11-06 19:59:02 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:59:02 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 19:59:54 in_model.py:236 step:12K smpl:198K ep:2K epch:75.39 loss:0.012 grdn:0.352 lr:6.5e-05 updt_s:1.017 data_s:0.021
+WARNING 2025-11-06 19:59:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 19:59:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:00:45 in_model.py:236 step:12K smpl:198K ep:2K epch:75.70 loss:0.012 grdn:0.332 lr:6.4e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 20:00:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:00:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:01:37 in_model.py:236 step:12K smpl:199K ep:2K epch:76.00 loss:0.013 grdn:0.336 lr:6.4e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 20:01:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:01:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:02:29 in_model.py:236 step:12K smpl:200K ep:2K epch:76.31 loss:0.011 grdn:0.306 lr:6.4e-05 updt_s:1.017 data_s:0.022
+WARNING 2025-11-06 20:02:29 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:02:29 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:03:20 in_model.py:236 step:13K smpl:201K ep:2K epch:76.61 loss:0.014 grdn:0.360 lr:6.4e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 20:03:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:03:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:04:11 in_model.py:236 step:13K smpl:202K ep:2K epch:76.92 loss:0.013 grdn:0.350 lr:6.3e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 20:04:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:04:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:05:03 in_model.py:236 step:13K smpl:202K ep:2K epch:77.22 loss:0.011 grdn:0.327 lr:6.3e-05 updt_s:1.015 data_s:0.021
+WARNING 2025-11-06 20:05:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:05:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:05:54 in_model.py:236 step:13K smpl:203K ep:2K epch:77.53 loss:0.012 grdn:0.306 lr:6.3e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 20:05:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:05:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:06:46 in_model.py:236 step:13K smpl:204K ep:2K epch:77.83 loss:0.014 grdn:0.341 lr:6.3e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 20:06:46 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:06:46 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:07:38 in_model.py:236 step:13K smpl:205K ep:2K epch:78.14 loss:0.012 grdn:0.332 lr:6.3e-05 updt_s:1.018 data_s:0.021
+WARNING 2025-11-06 20:07:38 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:07:38 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:08:29 in_model.py:236 step:13K smpl:206K ep:2K epch:78.44 loss:0.013 grdn:0.343 lr:6.2e-05 updt_s:1.019 data_s:0.000
+WARNING 2025-11-06 20:08:29 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:08:29 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:09:20 in_model.py:236 step:13K smpl:206K ep:2K epch:78.75 loss:0.012 grdn:0.343 lr:6.2e-05 updt_s:1.020 data_s:0.000
+WARNING 2025-11-06 20:09:20 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:09:20 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:10:11 in_model.py:236 step:13K smpl:207K ep:2K epch:79.05 loss:0.011 grdn:0.324 lr:6.2e-05 updt_s:1.023 data_s:0.000
+WARNING 2025-11-06 20:10:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:10:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:11:03 in_model.py:236 step:13K smpl:208K ep:2K epch:79.36 loss:0.012 grdn:0.344 lr:6.2e-05 updt_s:1.017 data_s:0.022
+WARNING 2025-11-06 20:11:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:11:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:11:55 in_model.py:236 step:13K smpl:209K ep:2K epch:79.66 loss:0.012 grdn:0.329 lr:6.1e-05 updt_s:1.022 data_s:0.000
+WARNING 2025-11-06 20:11:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:11:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:12:46 in_model.py:236 step:13K smpl:210K ep:2K epch:79.97 loss:0.010 grdn:0.311 lr:6.1e-05 updt_s:1.013 data_s:0.000
+WARNING 2025-11-06 20:12:46 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:12:46 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:13:37 in_model.py:236 step:13K smpl:210K ep:2K epch:80.27 loss:0.013 grdn:0.337 lr:6.1e-05 updt_s:1.011 data_s:0.020
+WARNING 2025-11-06 20:13:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:13:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:14:29 in_model.py:236 step:13K smpl:211K ep:2K epch:80.58 loss:0.013 grdn:0.316 lr:6.1e-05 updt_s:1.033 data_s:0.000
+WARNING 2025-11-06 20:14:29 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:14:29 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:15:22 in_model.py:236 step:13K smpl:212K ep:2K epch:80.89 loss:0.010 grdn:0.321 lr:6.0e-05 updt_s:1.051 data_s:0.000
+WARNING 2025-11-06 20:15:22 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:15:22 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:16:16 in_model.py:236 step:13K smpl:213K ep:2K epch:81.19 loss:0.013 grdn:0.370 lr:6.0e-05 updt_s:1.050 data_s:0.021
+WARNING 2025-11-06 20:16:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:16:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:17:08 in_model.py:236 step:13K smpl:214K ep:2K epch:81.50 loss:0.012 grdn:0.325 lr:6.0e-05 updt_s:1.051 data_s:0.000
+WARNING 2025-11-06 20:17:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:17:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:18:01 in_model.py:236 step:13K smpl:214K ep:2K epch:81.80 loss:0.012 grdn:0.298 lr:6.0e-05 updt_s:1.056 data_s:0.000
+WARNING 2025-11-06 20:18:01 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:18:01 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:18:55 in_model.py:236 step:13K smpl:215K ep:2K epch:82.11 loss:0.013 grdn:0.334 lr:5.9e-05 updt_s:1.052 data_s:0.020
+WARNING 2025-11-06 20:18:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:18:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:19:48 in_model.py:236 step:14K smpl:216K ep:2K epch:82.41 loss:0.013 grdn:0.316 lr:5.9e-05 updt_s:1.065 data_s:0.000
+WARNING 2025-11-06 20:19:48 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:19:48 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:20:42 in_model.py:236 step:14K smpl:217K ep:2K epch:82.72 loss:0.011 grdn:0.324 lr:5.9e-05 updt_s:1.065 data_s:0.000
+WARNING 2025-11-06 20:20:42 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:20:42 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:21:34 in_model.py:236 step:14K smpl:218K ep:2K epch:83.02 loss:0.014 grdn:0.359 lr:5.8e-05 updt_s:1.050 data_s:0.000
+WARNING 2025-11-06 20:21:34 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:21:34 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:22:27 in_model.py:236 step:14K smpl:218K ep:2K epch:83.33 loss:0.011 grdn:0.299 lr:5.8e-05 updt_s:1.032 data_s:0.021
+WARNING 2025-11-06 20:22:27 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:22:27 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:23:19 in_model.py:236 step:14K smpl:219K ep:2K epch:83.63 loss:0.010 grdn:0.307 lr:5.8e-05 updt_s:1.042 data_s:0.000
+WARNING 2025-11-06 20:23:19 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:23:19 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:24:11 in_model.py:236 step:14K smpl:220K ep:2K epch:83.94 loss:0.012 grdn:0.321 lr:5.8e-05 updt_s:1.027 data_s:0.000
+WARNING 2025-11-06 20:24:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:24:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:25:03 in_model.py:236 step:14K smpl:221K ep:2K epch:84.24 loss:0.012 grdn:0.334 lr:5.7e-05 updt_s:1.016 data_s:0.022
+WARNING 2025-11-06 20:25:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:25:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:25:54 in_model.py:236 step:14K smpl:222K ep:2K epch:84.55 loss:0.012 grdn:0.310 lr:5.7e-05 updt_s:1.016 data_s:0.000
+WARNING 2025-11-06 20:25:54 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:25:54 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:26:45 in_model.py:236 step:14K smpl:222K ep:2K epch:84.85 loss:0.011 grdn:0.295 lr:5.7e-05 updt_s:1.016 data_s:0.000
+WARNING 2025-11-06 20:26:45 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:26:45 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:27:36 in_model.py:236 step:14K smpl:223K ep:2K epch:85.16 loss:0.010 grdn:0.300 lr:5.7e-05 updt_s:1.006 data_s:0.021
+WARNING 2025-11-06 20:27:36 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:27:36 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:28:27 in_model.py:236 step:14K smpl:224K ep:2K epch:85.46 loss:0.009 grdn:0.272 lr:5.6e-05 updt_s:1.010 data_s:0.000
+WARNING 2025-11-06 20:28:27 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:28:27 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:29:18 in_model.py:236 step:14K smpl:225K ep:2K epch:85.77 loss:0.013 grdn:0.317 lr:5.6e-05 updt_s:1.010 data_s:0.000
+WARNING 2025-11-06 20:29:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:29:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:30:08 in_model.py:236 step:14K smpl:226K ep:2K epch:86.07 loss:0.014 grdn:0.377 lr:5.6e-05 updt_s:1.004 data_s:0.000
+WARNING 2025-11-06 20:30:08 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:30:08 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:30:59 in_model.py:236 step:14K smpl:226K ep:2K epch:86.38 loss:0.012 grdn:0.315 lr:5.6e-05 updt_s:1.000 data_s:0.021
+WARNING 2025-11-06 20:30:59 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:30:59 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:31:50 in_model.py:236 step:14K smpl:227K ep:2K epch:86.68 loss:0.009 grdn:0.279 lr:5.5e-05 updt_s:1.005 data_s:0.000
+WARNING 2025-11-06 20:31:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:31:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:32:40 in_model.py:236 step:14K smpl:228K ep:2K epch:86.99 loss:0.009 grdn:0.307 lr:5.5e-05 updt_s:1.005 data_s:0.000
+WARNING 2025-11-06 20:32:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:32:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:33:31 in_model.py:236 step:14K smpl:229K ep:2K epch:87.29 loss:0.011 grdn:0.283 lr:5.5e-05 updt_s:0.994 data_s:0.022
+WARNING 2025-11-06 20:33:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:33:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:34:21 in_model.py:236 step:14K smpl:230K ep:2K epch:87.60 loss:0.010 grdn:0.268 lr:5.5e-05 updt_s:0.996 data_s:0.000
+WARNING 2025-11-06 20:34:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:34:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:35:11 in_model.py:236 step:14K smpl:230K ep:2K epch:87.91 loss:0.010 grdn:0.304 lr:5.4e-05 updt_s:0.997 data_s:0.000
+WARNING 2025-11-06 20:35:11 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:35:11 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:36:02 in_model.py:236 step:14K smpl:231K ep:2K epch:88.21 loss:0.010 grdn:0.305 lr:5.4e-05 updt_s:1.001 data_s:0.022
+WARNING 2025-11-06 20:36:02 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:36:02 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:36:53 in_model.py:236 step:14K smpl:232K ep:2K epch:88.52 loss:0.014 grdn:0.371 lr:5.4e-05 updt_s:1.004 data_s:0.000
+WARNING 2025-11-06 20:36:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:36:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:37:43 in_model.py:236 step:15K smpl:233K ep:2K epch:88.82 loss:0.010 grdn:0.282 lr:5.4e-05 updt_s:1.005 data_s:0.000
+WARNING 2025-11-06 20:37:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:37:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:38:35 in_model.py:236 step:15K smpl:234K ep:2K epch:89.13 loss:0.009 grdn:0.285 lr:5.3e-05 updt_s:1.007 data_s:0.020
+WARNING 2025-11-06 20:38:35 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:38:35 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:39:25 in_model.py:236 step:15K smpl:234K ep:2K epch:89.43 loss:0.011 grdn:0.298 lr:5.3e-05 updt_s:1.009 data_s:0.000
+WARNING 2025-11-06 20:39:25 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:39:25 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:40:16 in_model.py:236 step:15K smpl:235K ep:2K epch:89.74 loss:0.009 grdn:0.265 lr:5.3e-05 updt_s:1.010 data_s:0.000
+WARNING 2025-11-06 20:40:16 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:40:16 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:41:07 in_model.py:236 step:15K smpl:236K ep:2K epch:90.04 loss:0.010 grdn:0.281 lr:5.3e-05 updt_s:1.013 data_s:0.000
+WARNING 2025-11-06 20:41:07 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:41:07 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:41:58 in_model.py:236 step:15K smpl:237K ep:2K epch:90.35 loss:0.011 grdn:0.331 lr:5.2e-05 updt_s:1.005 data_s:0.021
+WARNING 2025-11-06 20:41:58 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:41:58 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:42:49 in_model.py:236 step:15K smpl:238K ep:2K epch:90.65 loss:0.011 grdn:0.305 lr:5.2e-05 updt_s:1.013 data_s:0.000
+WARNING 2025-11-06 20:42:49 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:42:49 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:43:40 in_model.py:236 step:15K smpl:238K ep:2K epch:90.96 loss:0.008 grdn:0.234 lr:5.2e-05 updt_s:1.011 data_s:0.000
+WARNING 2025-11-06 20:43:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:43:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:44:31 in_model.py:236 step:15K smpl:239K ep:2K epch:91.26 loss:0.008 grdn:0.242 lr:5.2e-05 updt_s:1.003 data_s:0.021
+WARNING 2025-11-06 20:44:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:44:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:45:22 in_model.py:236 step:15K smpl:240K ep:2K epch:91.57 loss:0.011 grdn:0.326 lr:5.1e-05 updt_s:1.008 data_s:0.000
+WARNING 2025-11-06 20:45:22 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:45:22 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:45:22 in_model.py:245 Checkpoint policy after step 15000
+INFO 2025-11-06 20:46:15 in_model.py:236 step:15K smpl:241K ep:2K epch:91.87 loss:0.010 grdn:0.264 lr:5.1e-05 updt_s:0.996 data_s:0.000
+WARNING 2025-11-06 20:46:15 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:46:15 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:47:06 in_model.py:236 step:15K smpl:242K ep:2K epch:92.18 loss:0.013 grdn:0.347 lr:5.1e-05 updt_s:0.998 data_s:0.023
+WARNING 2025-11-06 20:47:06 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:47:06 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:47:57 in_model.py:236 step:15K smpl:242K ep:2K epch:92.48 loss:0.009 grdn:0.264 lr:5.1e-05 updt_s:1.000 data_s:0.000
+WARNING 2025-11-06 20:47:57 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:47:57 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:48:47 in_model.py:236 step:15K smpl:243K ep:2K epch:92.79 loss:0.010 grdn:0.306 lr:5.0e-05 updt_s:1.003 data_s:0.000
+WARNING 2025-11-06 20:48:47 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:48:47 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:49:37 in_model.py:236 step:15K smpl:244K ep:2K epch:93.09 loss:0.009 grdn:0.265 lr:5.0e-05 updt_s:1.003 data_s:0.000
+WARNING 2025-11-06 20:49:37 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:49:37 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:50:28 in_model.py:236 step:15K smpl:245K ep:2K epch:93.40 loss:0.011 grdn:0.304 lr:5.0e-05 updt_s:0.996 data_s:0.022
+WARNING 2025-11-06 20:50:28 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:50:28 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:51:18 in_model.py:236 step:15K smpl:246K ep:2K epch:93.70 loss:0.010 grdn:0.304 lr:5.0e-05 updt_s:1.003 data_s:0.000
+WARNING 2025-11-06 20:51:18 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:51:18 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:52:09 in_model.py:236 step:15K smpl:246K ep:2K epch:94.01 loss:0.010 grdn:0.299 lr:4.9e-05 updt_s:1.001 data_s:0.000
+WARNING 2025-11-06 20:52:09 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:52:09 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:53:00 in_model.py:236 step:15K smpl:247K ep:2K epch:94.32 loss:0.011 grdn:0.287 lr:4.9e-05 updt_s:0.996 data_s:0.021
+WARNING 2025-11-06 20:53:00 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:53:00 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:53:50 in_model.py:236 step:16K smpl:248K ep:2K epch:94.62 loss:0.012 grdn:0.330 lr:4.9e-05 updt_s:1.002 data_s:0.000
+WARNING 2025-11-06 20:53:50 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:53:50 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:54:40 in_model.py:236 step:16K smpl:249K ep:2K epch:94.93 loss:0.009 grdn:0.262 lr:4.9e-05 updt_s:1.002 data_s:0.000
+WARNING 2025-11-06 20:54:40 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:54:40 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:55:31 in_model.py:236 step:16K smpl:250K ep:2K epch:95.23 loss:0.010 grdn:0.262 lr:4.8e-05 updt_s:0.997 data_s:0.021
+WARNING 2025-11-06 20:55:31 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:55:31 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:56:21 in_model.py:236 step:16K smpl:250K ep:2K epch:95.54 loss:0.011 grdn:0.325 lr:4.8e-05 updt_s:1.003 data_s:0.000
+WARNING 2025-11-06 20:56:21 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:56:21 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:57:12 in_model.py:236 step:16K smpl:251K ep:2K epch:95.84 loss:0.011 grdn:0.304 lr:4.8e-05 updt_s:1.001 data_s:0.000
+WARNING 2025-11-06 20:57:12 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:57:12 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:58:03 in_model.py:236 step:16K smpl:252K ep:2K epch:96.15 loss:0.010 grdn:0.311 lr:4.8e-05 updt_s:0.996 data_s:0.021
+WARNING 2025-11-06 20:58:03 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:58:03 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:58:53 in_model.py:236 step:16K smpl:253K ep:2K epch:96.45 loss:0.008 grdn:0.273 lr:4.7e-05 updt_s:0.998 data_s:0.000
+WARNING 2025-11-06 20:58:53 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:58:53 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 20:59:43 in_model.py:236 step:16K smpl:254K ep:2K epch:96.76 loss:0.008 grdn:0.256 lr:4.7e-05 updt_s:0.997 data_s:0.000
+WARNING 2025-11-06 20:59:43 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 20:59:43 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 21:00:33 in_model.py:236 step:16K smpl:254K ep:2K epch:97.06 loss:0.008 grdn:0.231 lr:4.7e-05 updt_s:0.998 data_s:0.000
+WARNING 2025-11-06 21:00:33 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 21:00:33 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 21:01:24 in_model.py:236 step:16K smpl:255K ep:2K epch:97.37 loss:0.009 grdn:0.294 lr:4.7e-05 updt_s:0.994 data_s:0.021
+WARNING 2025-11-06 21:01:24 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 21:01:24 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 21:02:14 in_model.py:236 step:16K smpl:256K ep:2K epch:97.67 loss:0.009 grdn:0.257 lr:4.6e-05 updt_s:1.001 data_s:0.000
+WARNING 2025-11-06 21:02:14 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 21:02:14 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 21:03:04 in_model.py:236 step:16K smpl:257K ep:2K epch:97.98 loss:0.009 grdn:0.283 lr:4.6e-05 updt_s:1.000 data_s:0.000
+WARNING 2025-11-06 21:03:04 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 21:03:04 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+INFO 2025-11-06 21:03:55 in_model.py:236 step:16K smpl:258K ep:2K epch:98.28 loss:0.008 grdn:0.249 lr:4.6e-05 updt_s:0.995 data_s:0.022
+WARNING 2025-11-06 21:03:55 db_utils.py:117 WandB logging of key "losses_after_forward" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.
+WARNING 2025-11-06 21:03:55 db_utils.py:117 WandB logging of key "losses_after_rm_padding" was ignored as its type "<class 'torch.Tensor'>" is not handled by this wrapper.

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/logs/debug-internal.log

@@ -6,3 +6,4 @@
 {"time":"2025-11-06T16:21:50.381975054+08:00","level":"INFO","msg":"sender: started","stream_id":"8gs9of7r"}
 {"time":"2025-11-06T18:28:51.825582909+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
 {"time":"2025-11-06T19:15:51.940995974+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-11-06T20:24:22.8723501+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}

smolvla_omy/wandb/run-20251106_162149-8gs9of7r/run-8gs9of7r.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e86862bae404a6a86f44fbb8db08d5e5db202b135f2d98cd499831fa5bb85096
-size 1802240
+oid sha256:84e401863c9a52b1e0868f3395d571a429a67c767c565ab9b4c9e9d619e3ebc2
+size 2752512