Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250806_095353-4m3ikyrf/run-4m3ikyrf.wandb filter=lfs diff=lfs merge=lfs -text

checkpoints/020000/pretrained_model/config.json

@@ -0,0 +1,95 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.effort": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.images.cam_high": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.cam_left_wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "push_to_hub": true,
+    "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+    "private": null,
+    "tags": null,
+    "license": null,
+    "horizon": 16,
+    "n_action_steps": 8,
+    "drop_n_last_frames": 7,
+    "vision_backbone": "resnet18",
+    "crop_shape": [
+        84,
+        84
+    ],
+    "crop_is_random": true,
+    "pretrained_backbone_weights": null,
+    "use_group_norm": true,
+    "spatial_softmax_num_keypoints": 32,
+    "use_separate_rgb_encoder_per_camera": false,
+    "down_dims": [
+        512,
+        1024,
+        2048
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "noise_scheduler_type": "DDPM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": null,
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500
+}

checkpoints/020000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c399780d4f7c0ac81b0b3eef47ad81a0f0355294c8633688e7f5d357270355b
+size 1066761788

checkpoints/020000/pretrained_model/train_config.json

@@ -0,0 +1,205 @@
+{
+    "dataset": {
+        "repo_id": "a1o/pick-blue-cube_sarm",
+        "root": null,
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.effort": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.images.cam_high": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.cam_left_wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "push_to_hub": true,
+        "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+        "private": null,
+        "tags": null,
+        "license": null,
+        "horizon": 16,
+        "n_action_steps": 8,
+        "drop_n_last_frames": 7,
+        "vision_backbone": "resnet18",
+        "crop_shape": [
+            84,
+            84
+        ],
+        "crop_is_random": true,
+        "pretrained_backbone_weights": null,
+        "use_group_norm": true,
+        "spatial_softmax_num_keypoints": 32,
+        "use_separate_rgb_encoder_per_camera": false,
+        "down_dims": [
+            512,
+            1024,
+            2048
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "noise_scheduler_type": "DDPM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": null,
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500
+    },
+    "output_dir": "outputs/train/pick-blue-cube-single-arm_diff",
+    "job_name": "diff_pick-blue-cube-single-arm",
+    "resume": false,
+    "seed": 1000,
+    "num_workers": 4,
+    "batch_size": 8,
+    "steps": 100000,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "4m3ikyrf",
+        "mode": null
+    }
+}

checkpoints/020000/training_state/optimizer_param_groups.json

@@ -0,0 +1,233 @@
+[
+    {
+        "lr": 9.081878607071996e-05,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-06,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": false,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211
+        ]
+    }
+]

checkpoints/020000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4f0c86b12d1b574c3f451f55f25b5fd07685886dac99e760e891ef62e4571ff
+size 2133526512

checkpoints/020000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49f0d9220cb1fedc35646999d65c0c2a464f34c0338ec98ae1374a538ed19bf2
+size 15708

checkpoints/020000/training_state/scheduler_state.json

@@ -0,0 +1,14 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 20000,
+    "_step_count": 20001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        9.081878607071996e-05
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

checkpoints/020000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 20000
+}

checkpoints/040000/pretrained_model/config.json

@@ -0,0 +1,95 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.effort": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.images.cam_high": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.cam_left_wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "push_to_hub": true,
+    "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+    "private": null,
+    "tags": null,
+    "license": null,
+    "horizon": 16,
+    "n_action_steps": 8,
+    "drop_n_last_frames": 7,
+    "vision_backbone": "resnet18",
+    "crop_shape": [
+        84,
+        84
+    ],
+    "crop_is_random": true,
+    "pretrained_backbone_weights": null,
+    "use_group_norm": true,
+    "spatial_softmax_num_keypoints": 32,
+    "use_separate_rgb_encoder_per_camera": false,
+    "down_dims": [
+        512,
+        1024,
+        2048
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "noise_scheduler_type": "DDPM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": null,
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500
+}

checkpoints/040000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7beda125f073c30487fc021bd7422a690a5bc315865435cc9121dfcbc9d7bcc
+size 1066761788

checkpoints/040000/pretrained_model/train_config.json

@@ -0,0 +1,205 @@
+{
+    "dataset": {
+        "repo_id": "a1o/pick-blue-cube_sarm",
+        "root": null,
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.effort": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.images.cam_high": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.cam_left_wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "push_to_hub": true,
+        "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+        "private": null,
+        "tags": null,
+        "license": null,
+        "horizon": 16,
+        "n_action_steps": 8,
+        "drop_n_last_frames": 7,
+        "vision_backbone": "resnet18",
+        "crop_shape": [
+            84,
+            84
+        ],
+        "crop_is_random": true,
+        "pretrained_backbone_weights": null,
+        "use_group_norm": true,
+        "spatial_softmax_num_keypoints": 32,
+        "use_separate_rgb_encoder_per_camera": false,
+        "down_dims": [
+            512,
+            1024,
+            2048
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "noise_scheduler_type": "DDPM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": null,
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500
+    },
+    "output_dir": "outputs/train/pick-blue-cube-single-arm_diff",
+    "job_name": "diff_pick-blue-cube-single-arm",
+    "resume": false,
+    "seed": 1000,
+    "num_workers": 4,
+    "batch_size": 8,
+    "steps": 100000,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "4m3ikyrf",
+        "mode": null
+    }
+}

checkpoints/040000/training_state/optimizer_param_groups.json

@@ -0,0 +1,233 @@
+[
+    {
+        "lr": 6.590057681196191e-05,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-06,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": false,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211
+        ]
+    }
+]

checkpoints/040000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cc838d1977122278f7dd9522349bbcdea499a24fe759e4be509bcdc4f46bed2
+size 2133526512

checkpoints/040000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baadabd0e17dba2594cd6a0afbe27e050ae3846e17c4e4e9c4635ea93f8a0d2e
+size 15708

checkpoints/040000/training_state/scheduler_state.json

@@ -0,0 +1,14 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 40000,
+    "_step_count": 40001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        6.590057681196191e-05
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

checkpoints/040000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 40000
+}

checkpoints/060000/pretrained_model/config.json

@@ -0,0 +1,95 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.effort": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.images.cam_high": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.cam_left_wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "push_to_hub": true,
+    "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+    "private": null,
+    "tags": null,
+    "license": null,
+    "horizon": 16,
+    "n_action_steps": 8,
+    "drop_n_last_frames": 7,
+    "vision_backbone": "resnet18",
+    "crop_shape": [
+        84,
+        84
+    ],
+    "crop_is_random": true,
+    "pretrained_backbone_weights": null,
+    "use_group_norm": true,
+    "spatial_softmax_num_keypoints": 32,
+    "use_separate_rgb_encoder_per_camera": false,
+    "down_dims": [
+        512,
+        1024,
+        2048
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "noise_scheduler_type": "DDPM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": null,
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500
+}

checkpoints/060000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af80b2572fa1d564c47fcc1bbae667744bd62578d71abf4d0c59e5f55bdbfdef
+size 1066761788

checkpoints/060000/pretrained_model/train_config.json

@@ -0,0 +1,205 @@
+{
+    "dataset": {
+        "repo_id": "a1o/pick-blue-cube_sarm",
+        "root": null,
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.effort": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.images.cam_high": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.cam_left_wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "push_to_hub": true,
+        "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+        "private": null,
+        "tags": null,
+        "license": null,
+        "horizon": 16,
+        "n_action_steps": 8,
+        "drop_n_last_frames": 7,
+        "vision_backbone": "resnet18",
+        "crop_shape": [
+            84,
+            84
+        ],
+        "crop_is_random": true,
+        "pretrained_backbone_weights": null,
+        "use_group_norm": true,
+        "spatial_softmax_num_keypoints": 32,
+        "use_separate_rgb_encoder_per_camera": false,
+        "down_dims": [
+            512,
+            1024,
+            2048
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "noise_scheduler_type": "DDPM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": null,
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500
+    },
+    "output_dir": "outputs/train/pick-blue-cube-single-arm_diff",
+    "job_name": "diff_pick-blue-cube-single-arm",
+    "resume": false,
+    "seed": 1000,
+    "num_workers": 4,
+    "batch_size": 8,
+    "steps": 100000,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "4m3ikyrf",
+        "mode": null
+    }
+}

checkpoints/060000/training_state/optimizer_param_groups.json

@@ -0,0 +1,233 @@
+[
+    {
+        "lr": 3.484974098465636e-05,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-06,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": false,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211
+        ]
+    }
+]

checkpoints/060000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a61b2ae1f58541c6584edbe9c9344796ebb10539c402ba002c72c5307c8be31b
+size 2133526512

checkpoints/060000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d88b71cbb3ac2e055a9d2f0d54dff394e4f50eb9b20c78095a53ef5b9f8c272
+size 15708

checkpoints/060000/training_state/scheduler_state.json

@@ -0,0 +1,14 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 60000,
+    "_step_count": 60001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        3.484974098465636e-05
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

checkpoints/060000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 60000
+}

checkpoints/080000/pretrained_model/config.json

@@ -0,0 +1,95 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.effort": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.images.cam_high": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.cam_left_wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "push_to_hub": true,
+    "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+    "private": null,
+    "tags": null,
+    "license": null,
+    "horizon": 16,
+    "n_action_steps": 8,
+    "drop_n_last_frames": 7,
+    "vision_backbone": "resnet18",
+    "crop_shape": [
+        84,
+        84
+    ],
+    "crop_is_random": true,
+    "pretrained_backbone_weights": null,
+    "use_group_norm": true,
+    "spatial_softmax_num_keypoints": 32,
+    "use_separate_rgb_encoder_per_camera": false,
+    "down_dims": [
+        512,
+        1024,
+        2048
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "noise_scheduler_type": "DDPM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": null,
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500
+}

checkpoints/080000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a4307edd2c6b80d32295d0c14106d8c1001c5e74ab1969519bb356038b18ce
+size 1066761788

checkpoints/080000/pretrained_model/train_config.json

@@ -0,0 +1,205 @@
+{
+    "dataset": {
+        "repo_id": "a1o/pick-blue-cube_sarm",
+        "root": null,
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.effort": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.images.cam_high": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.cam_left_wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "push_to_hub": true,
+        "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+        "private": null,
+        "tags": null,
+        "license": null,
+        "horizon": 16,
+        "n_action_steps": 8,
+        "drop_n_last_frames": 7,
+        "vision_backbone": "resnet18",
+        "crop_shape": [
+            84,
+            84
+        ],
+        "crop_is_random": true,
+        "pretrained_backbone_weights": null,
+        "use_group_norm": true,
+        "spatial_softmax_num_keypoints": 32,
+        "use_separate_rgb_encoder_per_camera": false,
+        "down_dims": [
+            512,
+            1024,
+            2048
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "noise_scheduler_type": "DDPM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": null,
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500
+    },
+    "output_dir": "outputs/train/pick-blue-cube-single-arm_diff",
+    "job_name": "diff_pick-blue-cube-single-arm",
+    "resume": false,
+    "seed": 1000,
+    "num_workers": 4,
+    "batch_size": 8,
+    "steps": 100000,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "4m3ikyrf",
+        "mode": null
+    }
+}

checkpoints/080000/training_state/optimizer_param_groups.json

@@ -0,0 +1,233 @@
+[
+    {
+        "lr": 9.642144811900739e-06,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-06,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": false,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211
+        ]
+    }
+]

checkpoints/080000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0583ab84b3dfbd4eddfe1912da49599814f2f7256134774c2c08d0de25a2362e
+size 2133526512

checkpoints/080000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c3c2cffb203aefe19a519217a57cf8c90cd1516837b29090d958c0c12604284
+size 15708

checkpoints/080000/training_state/scheduler_state.json

@@ -0,0 +1,14 @@
+{
+    "base_lrs": [
+        0.0001
+    ],
+    "last_epoch": 80000,
+    "_step_count": 80001,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        9.642144811900739e-06
+    ],
+    "lr_lambdas": [
+        null
+    ]
+}

checkpoints/080000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 80000
+}

checkpoints/100000/pretrained_model/config.json

@@ -0,0 +1,95 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.effort": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.images.cam_high": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.cam_left_wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "push_to_hub": true,
+    "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+    "private": null,
+    "tags": null,
+    "license": null,
+    "horizon": 16,
+    "n_action_steps": 8,
+    "drop_n_last_frames": 7,
+    "vision_backbone": "resnet18",
+    "crop_shape": [
+        84,
+        84
+    ],
+    "crop_is_random": true,
+    "pretrained_backbone_weights": null,
+    "use_group_norm": true,
+    "spatial_softmax_num_keypoints": 32,
+    "use_separate_rgb_encoder_per_camera": false,
+    "down_dims": [
+        512,
+        1024,
+        2048
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "noise_scheduler_type": "DDPM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": null,
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500
+}

checkpoints/100000/pretrained_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a13893a1356ff503f3823a313e2abe19880ae2f4c2eb002c2d8b17382aa9370
+size 1066761788

checkpoints/100000/pretrained_model/train_config.json

@@ -0,0 +1,205 @@
+{
+    "dataset": {
+        "repo_id": "a1o/pick-blue-cube_sarm",
+        "root": null,
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.effort": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.images.cam_high": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.cam_left_wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "push_to_hub": true,
+        "repo_id": "a1o/diff_pick-blue-cube_sarm_policy",
+        "private": null,
+        "tags": null,
+        "license": null,
+        "horizon": 16,
+        "n_action_steps": 8,
+        "drop_n_last_frames": 7,
+        "vision_backbone": "resnet18",
+        "crop_shape": [
+            84,
+            84
+        ],
+        "crop_is_random": true,
+        "pretrained_backbone_weights": null,
+        "use_group_norm": true,
+        "spatial_softmax_num_keypoints": 32,
+        "use_separate_rgb_encoder_per_camera": false,
+        "down_dims": [
+            512,
+            1024,
+            2048
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "noise_scheduler_type": "DDPM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": null,
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500
+    },
+    "output_dir": "outputs/train/pick-blue-cube-single-arm_diff",
+    "job_name": "diff_pick-blue-cube-single-arm",
+    "resume": false,
+    "seed": 1000,
+    "num_workers": 4,
+    "batch_size": 8,
+    "steps": 100000,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "4m3ikyrf",
+        "mode": null
+    }
+}

checkpoints/100000/training_state/optimizer_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:138b88eaf4f14e48dcde8ae4b47ac9af56e42d27bbd9ed406ac629068f73bc80
+size 1006632960

checkpoints/100000/training_state/rng_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d744f34355be1c1e0fc3b499a26b13cec2a1a31684c17ef96b3e66943fe4cfe4
+size 15708

checkpoints/100000/training_state/training_step.json

@@ -0,0 +1,3 @@
+{
+    "step": 100000
+}

wandb/debug-internal.log

@@ -0,0 +1,36 @@
+{"time":"2025-08-06T09:53:53.97048859-05:00","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
+{"time":"2025-08-06T09:53:54.181541434-05:00","level":"INFO","msg":"stream: created new stream","id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.181593892-05:00","level":"INFO","msg":"stream: started","id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.182561483-05:00","level":"INFO","msg":"handler: started","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.182602292-05:00","level":"INFO","msg":"writer: Do: started","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.182749259-05:00","level":"INFO","msg":"sender: started","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T15:22:36.283609749-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:23:21.287384372-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:23:36.285724184-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:23:47.496306298-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:24:23.543803005-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/a10v-1/lerobot/4m3ikyrf/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:25:17.124550591-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/a10v-1/lerobot/4m3ikyrf/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:25:17.794337051-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:26:17.262055087-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:27:19.720003097-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:27:44.671942739-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:27:48.400231348-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/a10v-1/lerobot/4m3ikyrf/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:28:19.10820662-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:28:46.051038776-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:29:57.280536859-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-06T15:30:20.121098493-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:31:42.281292653-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-06T15:32:05.208409505-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:32:29.220382351-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:33:16.719371795-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:33:39.286298851-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:37:44.31879675-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:40:17.910537945-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:48:57.304808267-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-06T16:25:37.13628613-05:00","level":"ERROR","msg":"error adding file to cache","err":"write /home/aye8078/.cache/wandb/artifacts/tmp/2102143582: disk quota exceeded"}
+{"time":"2025-08-06T18:01:25.622284787-05:00","level":"INFO","msg":"stream: closing","id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.32138818-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-06T18:01:26.414301344-05:00","level":"INFO","msg":"handler: closed","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.414318337-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.414434409-05:00","level":"INFO","msg":"sender: closed","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.485367694-05:00","level":"INFO","msg":"stream: closed","id":"4m3ikyrf"}

wandb/debug.log

@@ -0,0 +1,22 @@
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Configure stats pid to 4150232
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Loading settings from /home/aye8078/.config/wandb/settings
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Loading settings from /gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/wandb/settings
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/pick-blue-cube-single-arm_diff/wandb/run-20250806_095353-4m3ikyrf/logs/debug.log
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/pick-blue-cube-single-arm_diff/wandb/run-20250806_095353-4m3ikyrf/logs/debug-internal.log
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:init():830] calling init triggers
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'a1o/pick-blue-cube_sarm', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MIN_MAX: 'MIN_MAX'>, 'ACTION': <NormalizationMode.MIN_MAX: 'MIN_MAX'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'push_to_hub': True, 'repo_id': 'a1o/diff_pick-blue-cube_sarm_policy', 'private': None, 'tags': None, 'license': None, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet18', 'crop_shape': [84, 84], 'crop_is_random': True, 'pretrained_backbone_weights': None, 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [512, 1024, 2048], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDPM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': None, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500}, 'output_dir': 'outputs/train/pick-blue-cube-single-arm_diff', 'job_name': 'diff_pick-blue-cube-single-arm', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 100000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
+2025-08-06 09:53:53,190 INFO    MainThread:4150232 [wandb_init.py:init():871] starting backend
+2025-08-06 09:53:53,911 INFO    MainThread:4150232 [wandb_init.py:init():874] sending inform_init request
+2025-08-06 09:53:53,965 INFO    MainThread:4150232 [wandb_init.py:init():882] backend started and connected
+2025-08-06 09:53:53,966 INFO    MainThread:4150232 [wandb_init.py:init():953] updated telemetry
+2025-08-06 09:53:53,966 INFO    MainThread:4150232 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-06 09:53:54,373 INFO    MainThread:4150232 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_console_start():2458] atexit reg
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_redirect():2375] Wrapping output streams.
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_redirect():2398] Redirects installed.
+2025-08-06 09:53:55,940 INFO    MainThread:4150232 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-06 18:01:25,533 INFO    MsgRouterThr:4150232 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

wandb/run-20250806_095353-4m3ikyrf/files/config.yaml

@@ -0,0 +1,253 @@
+_wandb:
+    value:
+        cli_version: 0.21.0
+        e:
+            e37fw9fep90im526hixlw3wga8gghk11:
+                args:
+                    - --dataset.repo_id=a1o/pick-blue-cube_sarm
+                    - --policy.type=diffusion
+                    - --output_dir=outputs/train/pick-blue-cube-single-arm_diff
+                    - --job_name=diff_pick-blue-cube-single-arm
+                    - --policy.device=cuda
+                    - --wandb.enable=true
+                    - --policy.repo_id=a1o/diff_pick-blue-cube_sarm_policy
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "540905492480"
+                        used: "7185981440"
+                executable: /projects/p32775/pythonenvs/tac_vla/bin/python
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-8a97223c-63d6-86ef-9b46-7e4225e9933e
+                host: qgpu3019
+                memory:
+                    total: "1081810989056"
+                os: Linux-4.18.0-553.36.1.el8_10.x86_64-x86_64-with-glibc2.28
+                program: -m lerobot.scripts.train
+                python: CPython 3.10.18
+                root: outputs/train/pick-blue-cube-single-arm_diff
+                slurm:
+                    cluster_name: quest
+                    conf: /etc/slurm/slurm.conf
+                    cpus_on_node: "1"
+                    gpus_on_node: "1"
+                    gtids: "0"
+                    job_account: p32775
+                    job_cpus_per_node: "1"
+                    job_end_time: "1754527940"
+                    job_gid: "2000033"
+                    job_gpus: "1"
+                    job_id: "971322"
+                    job_name: ACT_pick-blue-cube-single-arm
+                    job_nodelist: qgpu3019
+                    job_num_nodes: "1"
+                    job_partition: gengpu
+                    job_qos: normal
+                    job_start_time: "1754491940"
+                    job_uid: "2000033"
+                    job_user: aye8078
+                    jobid: "971322"
+                    localid: "0"
+                    mem_per_node: "49152"
+                    nnodes: "1"
+                    nodeid: "0"
+                    nodelist: qgpu3019
+                    prio_process: "0"
+                    procid: "0"
+                    submit_dir: /gpfs/home/aye8078/Documents/Github/tac_vla/slurm
+                    submit_host: quser32
+                    task_pid: "4150120"
+                    tasks_per_node: "1"
+                    topology_addr: qgpu3019
+                    topology_addr_pattern: node
+                startedAt: "2025-08-06T14:53:53.171298Z"
+                writerId: e37fw9fep90im526hixlw3wga8gghk11
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 51
+            "2":
+                - 1
+                - 41
+                - 49
+                - 51
+                - 83
+            "3":
+                - 13
+                - 15
+                - 16
+                - 61
+            "4": 3.10.18
+            "5": 0.21.0
+            "10":
+                - 21
+            "12": 0.21.0
+            "13": linux-x86_64
+batch_size:
+    value: 8
+dataset:
+    value:
+        episodes: null
+        image_transforms:
+            enable: false
+            max_num_transforms: 3
+            random_order: false
+            tfs:
+                brightness:
+                    kwargs:
+                        brightness:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                contrast:
+                    kwargs:
+                        contrast:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                hue:
+                    kwargs:
+                        hue:
+                            - -0.05
+                            - 0.05
+                    type: ColorJitter
+                    weight: 1
+                saturation:
+                    kwargs:
+                        saturation:
+                            - 0.5
+                            - 1.5
+                    type: ColorJitter
+                    weight: 1
+                sharpness:
+                    kwargs:
+                        sharpness:
+                            - 0.5
+                            - 1.5
+                    type: SharpnessJitter
+                    weight: 1
+        repo_id: a1o/pick-blue-cube_sarm
+        revision: null
+        root: null
+        use_imagenet_stats: true
+        video_backend: torchcodec
+env:
+    value: null
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        use_async_envs: false
+eval_freq:
+    value: 20000
+job_name:
+    value: diff_pick-blue-cube-single-arm
+log_freq:
+    value: 200
+num_workers:
+    value: 4
+optimizer:
+    value:
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        grad_clip_norm: 10
+        lr: 0.0001
+        type: adam
+        weight_decay: 1e-06
+output_dir:
+    value: outputs/train/pick-blue-cube-single-arm_diff
+policy:
+    value:
+        beta_end: 0.02
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        clip_sample: true
+        clip_sample_range: 1
+        crop_is_random: true
+        crop_shape:
+            - 84
+            - 84
+        device: cuda
+        diffusion_step_embed_dim: 128
+        do_mask_loss_for_padding: false
+        down_dims:
+            - 512
+            - 1024
+            - 2048
+        drop_n_last_frames: 7
+        horizon: 16
+        kernel_size: 5
+        license: null
+        n_action_steps: 8
+        n_groups: 8
+        n_obs_steps: 2
+        noise_scheduler_type: DDPM
+        normalization_mapping:
+            ACTION: MIN_MAX
+            STATE: MIN_MAX
+            VISUAL: MEAN_STD
+        num_inference_steps: null
+        num_train_timesteps: 100
+        optimizer_betas:
+            - 0.95
+            - 0.999
+        optimizer_eps: 1e-08
+        optimizer_lr: 0.0001
+        optimizer_weight_decay: 1e-06
+        prediction_type: epsilon
+        pretrained_backbone_weights: null
+        private: null
+        push_to_hub: true
+        repo_id: a1o/diff_pick-blue-cube_sarm_policy
+        scheduler_name: cosine
+        scheduler_warmup_steps: 500
+        spatial_softmax_num_keypoints: 32
+        tags: null
+        type: diffusion
+        use_amp: false
+        use_film_scale_modulation: true
+        use_group_norm: true
+        use_separate_rgb_encoder_per_camera: false
+        vision_backbone: resnet18
+resume:
+    value: false
+save_checkpoint:
+    value: true
+save_freq:
+    value: 20000
+scheduler:
+    value:
+        name: cosine
+        num_warmup_steps: 500
+        type: diffuser
+seed:
+    value: 1000
+steps:
+    value: 100000
+use_policy_training_preset:
+    value: true
+wandb:
+    value:
+        disable_artifact: false
+        enable: true
+        entity: null
+        mode: null
+        notes: null
+        project: lerobot
+        run_id: null

wandb/run-20250806_095353-4m3ikyrf/files/output.log

@@ -0,0 +1,543 @@
+Logs will be synced with wandb.
+INFO 2025-08-06 09:53:55 db_utils.py:103 Track this run --> https://wandb.ai/a10v-1/lerobot/runs/4m3ikyrf
+INFO 2025-08-06 09:53:56 ts/train.py:127 Creating dataset
+INFO 2025-08-06 09:53:59 ts/train.py:138 Creating policy
+INFO 2025-08-06 09:54:02 ts/train.py:144 Creating optimizer and scheduler
+INFO 2025-08-06 09:54:02 ts/train.py:156 Output dir: outputs/train/pick-blue-cube-single-arm_diff
+INFO 2025-08-06 09:54:02 ts/train.py:159 cfg.steps=100000 (100K)
+INFO 2025-08-06 09:54:02 ts/train.py:160 dataset.num_frames=20000 (20K)
+INFO 2025-08-06 09:54:02 ts/train.py:161 dataset.num_episodes=50
+INFO 2025-08-06 09:54:02 ts/train.py:162 num_learnable_params=266683687 (267M)
+INFO 2025-08-06 09:54:02 ts/train.py:163 num_total_params=266683755 (267M)
+/projects/p32775/pythonenvs/tac_vla/lib/python3.10/site-packages/torch/utils/data/dataloader.py:626: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 1, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
+  warnings.warn(
+INFO 2025-08-06 09:54:02 ts/train.py:202 Start offline training on a fixed dataset
+INFO 2025-08-06 09:55:09 ts/train.py:232 step:200 smpl:2K ep:4 epch:0.08 loss:0.744 grdn:5.628 lr:2.0e-05 updt_s:0.101 data_s:0.234
+INFO 2025-08-06 09:56:07 ts/train.py:232 step:400 smpl:3K ep:8 epch:0.16 loss:0.171 grdn:3.399 lr:6.0e-05 updt_s:0.077 data_s:0.215
+INFO 2025-08-06 09:57:05 ts/train.py:232 step:600 smpl:5K ep:12 epch:0.24 loss:0.122 grdn:2.083 lr:9.5e-05 updt_s:0.079 data_s:0.211
+INFO 2025-08-06 09:58:04 ts/train.py:232 step:800 smpl:6K ep:16 epch:0.32 loss:0.083 grdn:1.366 lr:1.0e-04 updt_s:0.073 data_s:0.217
+INFO 2025-08-06 09:59:02 ts/train.py:232 step:1K smpl:8K ep:20 epch:0.40 loss:0.073 grdn:1.213 lr:1.0e-04 updt_s:0.080 data_s:0.211
+INFO 2025-08-06 10:00:00 ts/train.py:232 step:1K smpl:10K ep:24 epch:0.48 loss:0.066 grdn:1.090 lr:1.0e-04 updt_s:0.078 data_s:0.212
+INFO 2025-08-06 10:00:58 ts/train.py:232 step:1K smpl:11K ep:28 epch:0.56 loss:0.065 grdn:1.004 lr:1.0e-04 updt_s:0.080 data_s:0.211
+INFO 2025-08-06 10:01:56 ts/train.py:232 step:2K smpl:13K ep:32 epch:0.64 loss:0.052 grdn:0.860 lr:1.0e-04 updt_s:0.081 data_s:0.209
+INFO 2025-08-06 10:02:55 ts/train.py:232 step:2K smpl:14K ep:36 epch:0.72 loss:0.050 grdn:0.856 lr:1.0e-04 updt_s:0.081 data_s:0.211
+INFO 2025-08-06 10:03:53 ts/train.py:232 step:2K smpl:16K ep:40 epch:0.80 loss:0.051 grdn:0.819 lr:1.0e-04 updt_s:0.080 data_s:0.210
+INFO 2025-08-06 10:04:51 ts/train.py:232 step:2K smpl:18K ep:44 epch:0.88 loss:0.046 grdn:0.786 lr:1.0e-04 updt_s:0.076 data_s:0.215
+INFO 2025-08-06 10:05:49 ts/train.py:232 step:2K smpl:19K ep:48 epch:0.96 loss:0.052 grdn:0.844 lr:1.0e-04 updt_s:0.078 data_s:0.212
+INFO 2025-08-06 10:07:02 ts/train.py:232 step:3K smpl:21K ep:52 epch:1.04 loss:0.049 grdn:0.765 lr:1.0e-04 updt_s:0.129 data_s:0.236
+INFO 2025-08-06 10:08:01 ts/train.py:232 step:3K smpl:22K ep:56 epch:1.12 loss:0.040 grdn:0.684 lr:1.0e-04 updt_s:0.059 data_s:0.234
+INFO 2025-08-06 10:09:00 ts/train.py:232 step:3K smpl:24K ep:60 epch:1.20 loss:0.041 grdn:0.690 lr:1.0e-04 updt_s:0.070 data_s:0.225
+INFO 2025-08-06 10:09:59 ts/train.py:232 step:3K smpl:26K ep:64 epch:1.28 loss:0.045 grdn:0.715 lr:1.0e-04 updt_s:0.058 data_s:0.235
+INFO 2025-08-06 10:10:58 ts/train.py:232 step:3K smpl:27K ep:68 epch:1.36 loss:0.043 grdn:0.666 lr:1.0e-04 updt_s:0.062 data_s:0.233
+INFO 2025-08-06 10:11:57 ts/train.py:232 step:4K smpl:29K ep:72 epch:1.44 loss:0.040 grdn:0.659 lr:1.0e-04 updt_s:0.064 data_s:0.230
+INFO 2025-08-06 10:12:56 ts/train.py:232 step:4K smpl:30K ep:76 epch:1.52 loss:0.040 grdn:0.635 lr:1.0e-04 updt_s:0.063 data_s:0.232
+INFO 2025-08-06 10:13:55 ts/train.py:232 step:4K smpl:32K ep:80 epch:1.60 loss:0.040 grdn:0.626 lr:1.0e-04 updt_s:0.065 data_s:0.229
+INFO 2025-08-06 10:14:54 ts/train.py:232 step:4K smpl:34K ep:84 epch:1.68 loss:0.036 grdn:0.604 lr:1.0e-04 updt_s:0.064 data_s:0.230
+INFO 2025-08-06 10:15:53 ts/train.py:232 step:4K smpl:35K ep:88 epch:1.76 loss:0.037 grdn:0.616 lr:1.0e-04 updt_s:0.058 data_s:0.235
+INFO 2025-08-06 10:16:51 ts/train.py:232 step:5K smpl:37K ep:92 epch:1.84 loss:0.036 grdn:0.594 lr:1.0e-04 updt_s:0.060 data_s:0.233
+INFO 2025-08-06 10:17:50 ts/train.py:232 step:5K smpl:38K ep:96 epch:1.92 loss:0.039 grdn:0.584 lr:1.0e-04 updt_s:0.053 data_s:0.242
+INFO 2025-08-06 10:18:50 ts/train.py:232 step:5K smpl:40K ep:100 epch:2.00 loss:0.034 grdn:0.569 lr:1.0e-04 updt_s:0.063 data_s:0.235
+INFO 2025-08-06 10:19:50 ts/train.py:232 step:5K smpl:42K ep:104 epch:2.08 loss:0.035 grdn:0.578 lr:9.9e-05 updt_s:0.078 data_s:0.221
+INFO 2025-08-06 10:20:50 ts/train.py:232 step:5K smpl:43K ep:108 epch:2.16 loss:0.034 grdn:0.547 lr:9.9e-05 updt_s:0.082 data_s:0.216
+INFO 2025-08-06 10:21:49 ts/train.py:232 step:6K smpl:45K ep:112 epch:2.24 loss:0.035 grdn:0.530 lr:9.9e-05 updt_s:0.081 data_s:0.215
+INFO 2025-08-06 10:22:48 ts/train.py:232 step:6K smpl:46K ep:116 epch:2.32 loss:0.037 grdn:0.571 lr:9.9e-05 updt_s:0.081 data_s:0.215
+INFO 2025-08-06 10:23:48 ts/train.py:232 step:6K smpl:48K ep:120 epch:2.40 loss:0.034 grdn:0.558 lr:9.9e-05 updt_s:0.081 data_s:0.216
+INFO 2025-08-06 10:24:47 ts/train.py:232 step:6K smpl:50K ep:124 epch:2.48 loss:0.031 grdn:0.496 lr:9.9e-05 updt_s:0.082 data_s:0.214
+INFO 2025-08-06 10:25:46 ts/train.py:232 step:6K smpl:51K ep:128 epch:2.56 loss:0.037 grdn:0.545 lr:9.9e-05 updt_s:0.080 data_s:0.216
+INFO 2025-08-06 10:26:46 ts/train.py:232 step:7K smpl:53K ep:132 epch:2.64 loss:0.034 grdn:0.517 lr:9.9e-05 updt_s:0.081 data_s:0.214
+INFO 2025-08-06 10:27:45 ts/train.py:232 step:7K smpl:54K ep:136 epch:2.72 loss:0.032 grdn:0.460 lr:9.9e-05 updt_s:0.081 data_s:0.215
+INFO 2025-08-06 10:28:44 ts/train.py:232 step:7K smpl:56K ep:140 epch:2.80 loss:0.031 grdn:0.485 lr:9.9e-05 updt_s:0.081 data_s:0.217
+INFO 2025-08-06 10:29:44 ts/train.py:232 step:7K smpl:58K ep:144 epch:2.88 loss:0.033 grdn:0.496 lr:9.9e-05 updt_s:0.082 data_s:0.213
+INFO 2025-08-06 10:30:43 ts/train.py:232 step:7K smpl:59K ep:148 epch:2.96 loss:0.031 grdn:0.493 lr:9.9e-05 updt_s:0.078 data_s:0.219
+INFO 2025-08-06 10:31:42 ts/train.py:232 step:8K smpl:61K ep:152 epch:3.04 loss:0.031 grdn:0.476 lr:9.9e-05 updt_s:0.070 data_s:0.223
+INFO 2025-08-06 10:32:40 ts/train.py:232 step:8K smpl:62K ep:156 epch:3.12 loss:0.028 grdn:0.463 lr:9.9e-05 updt_s:0.079 data_s:0.212
+INFO 2025-08-06 10:33:38 ts/train.py:232 step:8K smpl:64K ep:160 epch:3.20 loss:0.031 grdn:0.478 lr:9.9e-05 updt_s:0.079 data_s:0.210
+INFO 2025-08-06 10:34:36 ts/train.py:232 step:8K smpl:66K ep:164 epch:3.28 loss:0.031 grdn:0.490 lr:9.9e-05 updt_s:0.079 data_s:0.211
+INFO 2025-08-06 10:35:35 ts/train.py:232 step:8K smpl:67K ep:168 epch:3.36 loss:0.033 grdn:0.472 lr:9.8e-05 updt_s:0.078 data_s:0.215
+INFO 2025-08-06 10:36:33 ts/train.py:232 step:9K smpl:69K ep:172 epch:3.44 loss:0.032 grdn:0.471 lr:9.8e-05 updt_s:0.078 data_s:0.212
+INFO 2025-08-06 10:37:31 ts/train.py:232 step:9K smpl:70K ep:176 epch:3.52 loss:0.031 grdn:0.455 lr:9.8e-05 updt_s:0.079 data_s:0.211
+INFO 2025-08-06 10:38:29 ts/train.py:232 step:9K smpl:72K ep:180 epch:3.60 loss:0.030 grdn:0.446 lr:9.8e-05 updt_s:0.078 data_s:0.211
+INFO 2025-08-06 10:39:27 ts/train.py:232 step:9K smpl:74K ep:184 epch:3.68 loss:0.029 grdn:0.437 lr:9.8e-05 updt_s:0.080 data_s:0.210
+INFO 2025-08-06 10:40:26 ts/train.py:232 step:9K smpl:75K ep:188 epch:3.76 loss:0.029 grdn:0.444 lr:9.8e-05 updt_s:0.079 data_s:0.212
+INFO 2025-08-06 10:41:24 ts/train.py:232 step:10K smpl:77K ep:192 epch:3.84 loss:0.029 grdn:0.438 lr:9.8e-05 updt_s:0.080 data_s:0.210
+INFO 2025-08-06 10:42:22 ts/train.py:232 step:10K smpl:78K ep:196 epch:3.92 loss:0.029 grdn:0.435 lr:9.8e-05 updt_s:0.080 data_s:0.210
+INFO 2025-08-06 10:43:21 ts/train.py:232 step:10K smpl:80K ep:200 epch:4.00 loss:0.029 grdn:0.454 lr:9.8e-05 updt_s:0.047 data_s:0.248
+INFO 2025-08-06 10:44:21 ts/train.py:232 step:10K smpl:82K ep:204 epch:4.08 loss:0.029 grdn:0.442 lr:9.8e-05 updt_s:0.047 data_s:0.251
+INFO 2025-08-06 10:45:20 ts/train.py:232 step:10K smpl:83K ep:208 epch:4.16 loss:0.027 grdn:0.424 lr:9.8e-05 updt_s:0.064 data_s:0.234
+INFO 2025-08-06 10:46:20 ts/train.py:232 step:11K smpl:85K ep:212 epch:4.24 loss:0.027 grdn:0.430 lr:9.8e-05 updt_s:0.075 data_s:0.222
+INFO 2025-08-06 10:47:19 ts/train.py:232 step:11K smpl:86K ep:216 epch:4.32 loss:0.025 grdn:0.401 lr:9.7e-05 updt_s:0.074 data_s:0.223
+INFO 2025-08-06 10:48:19 ts/train.py:232 step:11K smpl:88K ep:220 epch:4.40 loss:0.029 grdn:0.437 lr:9.7e-05 updt_s:0.080 data_s:0.217
+INFO 2025-08-06 10:49:18 ts/train.py:232 step:11K smpl:90K ep:224 epch:4.48 loss:0.030 grdn:0.446 lr:9.7e-05 updt_s:0.079 data_s:0.217
+INFO 2025-08-06 10:50:18 ts/train.py:232 step:11K smpl:91K ep:228 epch:4.56 loss:0.029 grdn:0.412 lr:9.7e-05 updt_s:0.077 data_s:0.220
+INFO 2025-08-06 10:51:17 ts/train.py:232 step:12K smpl:93K ep:232 epch:4.64 loss:0.029 grdn:0.419 lr:9.7e-05 updt_s:0.077 data_s:0.221
+INFO 2025-08-06 10:52:17 ts/train.py:232 step:12K smpl:94K ep:236 epch:4.72 loss:0.027 grdn:0.413 lr:9.7e-05 updt_s:0.075 data_s:0.222
+INFO 2025-08-06 10:53:16 ts/train.py:232 step:12K smpl:96K ep:240 epch:4.80 loss:0.028 grdn:0.427 lr:9.7e-05 updt_s:0.079 data_s:0.216
+INFO 2025-08-06 10:54:16 ts/train.py:232 step:12K smpl:98K ep:244 epch:4.88 loss:0.028 grdn:0.428 lr:9.7e-05 updt_s:0.080 data_s:0.217
+INFO 2025-08-06 10:55:15 ts/train.py:232 step:12K smpl:99K ep:248 epch:4.96 loss:0.028 grdn:0.423 lr:9.7e-05 updt_s:0.068 data_s:0.231
+INFO 2025-08-06 10:56:15 ts/train.py:232 step:13K smpl:101K ep:252 epch:5.04 loss:0.029 grdn:0.425 lr:9.6e-05 updt_s:0.075 data_s:0.222
+INFO 2025-08-06 10:57:14 ts/train.py:232 step:13K smpl:102K ep:256 epch:5.12 loss:0.026 grdn:0.430 lr:9.6e-05 updt_s:0.075 data_s:0.222
+INFO 2025-08-06 10:58:13 ts/train.py:232 step:13K smpl:104K ep:260 epch:5.20 loss:0.027 grdn:0.423 lr:9.6e-05 updt_s:0.070 data_s:0.224
+INFO 2025-08-06 10:59:13 ts/train.py:232 step:13K smpl:106K ep:264 epch:5.28 loss:0.026 grdn:0.405 lr:9.6e-05 updt_s:0.069 data_s:0.227
+INFO 2025-08-06 11:00:12 ts/train.py:232 step:13K smpl:107K ep:268 epch:5.36 loss:0.027 grdn:0.409 lr:9.6e-05 updt_s:0.078 data_s:0.218
+INFO 2025-08-06 11:01:12 ts/train.py:232 step:14K smpl:109K ep:272 epch:5.44 loss:0.025 grdn:0.385 lr:9.6e-05 updt_s:0.078 data_s:0.219
+INFO 2025-08-06 11:02:11 ts/train.py:232 step:14K smpl:110K ep:276 epch:5.52 loss:0.025 grdn:0.382 lr:9.6e-05 updt_s:0.076 data_s:0.221
+INFO 2025-08-06 11:03:10 ts/train.py:232 step:14K smpl:112K ep:280 epch:5.60 loss:0.025 grdn:0.402 lr:9.6e-05 updt_s:0.075 data_s:0.219
+INFO 2025-08-06 11:04:10 ts/train.py:232 step:14K smpl:114K ep:284 epch:5.68 loss:0.026 grdn:0.403 lr:9.5e-05 updt_s:0.070 data_s:0.225
+INFO 2025-08-06 11:05:09 ts/train.py:232 step:14K smpl:115K ep:288 epch:5.76 loss:0.026 grdn:0.407 lr:9.5e-05 updt_s:0.071 data_s:0.225
+INFO 2025-08-06 11:06:08 ts/train.py:232 step:15K smpl:117K ep:292 epch:5.84 loss:0.024 grdn:0.393 lr:9.5e-05 updt_s:0.065 data_s:0.231
+INFO 2025-08-06 11:07:09 ts/train.py:232 step:15K smpl:118K ep:296 epch:5.92 loss:0.025 grdn:0.406 lr:9.5e-05 updt_s:0.059 data_s:0.245
+INFO 2025-08-06 11:08:08 ts/train.py:232 step:15K smpl:120K ep:300 epch:6.00 loss:0.025 grdn:0.400 lr:9.5e-05 updt_s:0.078 data_s:0.215
+INFO 2025-08-06 11:09:06 ts/train.py:232 step:15K smpl:122K ep:304 epch:6.08 loss:0.025 grdn:0.385 lr:9.5e-05 updt_s:0.075 data_s:0.217
+INFO 2025-08-06 11:10:05 ts/train.py:232 step:15K smpl:123K ep:308 epch:6.16 loss:0.024 grdn:0.408 lr:9.5e-05 updt_s:0.079 data_s:0.211
+INFO 2025-08-06 11:11:03 ts/train.py:232 step:16K smpl:125K ep:312 epch:6.24 loss:0.028 grdn:0.444 lr:9.4e-05 updt_s:0.077 data_s:0.213
+INFO 2025-08-06 11:12:01 ts/train.py:232 step:16K smpl:126K ep:316 epch:6.32 loss:0.024 grdn:0.368 lr:9.4e-05 updt_s:0.069 data_s:0.222
+INFO 2025-08-06 11:12:59 ts/train.py:232 step:16K smpl:128K ep:320 epch:6.40 loss:0.026 grdn:0.414 lr:9.4e-05 updt_s:0.067 data_s:0.224
+INFO 2025-08-06 11:13:58 ts/train.py:232 step:16K smpl:130K ep:324 epch:6.48 loss:0.024 grdn:0.389 lr:9.4e-05 updt_s:0.071 data_s:0.222
+INFO 2025-08-06 11:14:57 ts/train.py:232 step:16K smpl:131K ep:328 epch:6.56 loss:0.024 grdn:0.379 lr:9.4e-05 updt_s:0.075 data_s:0.216
+INFO 2025-08-06 11:15:55 ts/train.py:232 step:17K smpl:133K ep:332 epch:6.64 loss:0.024 grdn:0.378 lr:9.4e-05 updt_s:0.076 data_s:0.215
+INFO 2025-08-06 11:16:53 ts/train.py:232 step:17K smpl:134K ep:336 epch:6.72 loss:0.027 grdn:0.413 lr:9.4e-05 updt_s:0.077 data_s:0.215
+INFO 2025-08-06 11:17:52 ts/train.py:232 step:17K smpl:136K ep:340 epch:6.80 loss:0.024 grdn:0.378 lr:9.3e-05 updt_s:0.077 data_s:0.214
+INFO 2025-08-06 11:18:51 ts/train.py:232 step:17K smpl:138K ep:344 epch:6.88 loss:0.026 grdn:0.392 lr:9.3e-05 updt_s:0.075 data_s:0.222
+INFO 2025-08-06 11:19:51 ts/train.py:232 step:17K smpl:139K ep:348 epch:6.96 loss:0.023 grdn:0.380 lr:9.3e-05 updt_s:0.073 data_s:0.224
+INFO 2025-08-06 11:20:50 ts/train.py:232 step:18K smpl:141K ep:352 epch:7.04 loss:0.024 grdn:0.403 lr:9.3e-05 updt_s:0.078 data_s:0.217
+INFO 2025-08-06 11:21:49 ts/train.py:232 step:18K smpl:142K ep:356 epch:7.12 loss:0.022 grdn:0.377 lr:9.3e-05 updt_s:0.081 data_s:0.214
+INFO 2025-08-06 11:22:47 ts/train.py:232 step:18K smpl:144K ep:360 epch:7.20 loss:0.025 grdn:0.393 lr:9.3e-05 updt_s:0.079 data_s:0.214
+INFO 2025-08-06 11:23:47 ts/train.py:232 step:18K smpl:146K ep:364 epch:7.28 loss:0.024 grdn:0.389 lr:9.2e-05 updt_s:0.080 data_s:0.215
+INFO 2025-08-06 11:24:45 ts/train.py:232 step:18K smpl:147K ep:368 epch:7.36 loss:0.026 grdn:0.413 lr:9.2e-05 updt_s:0.080 data_s:0.212
+INFO 2025-08-06 11:25:44 ts/train.py:232 step:19K smpl:149K ep:372 epch:7.44 loss:0.024 grdn:0.388 lr:9.2e-05 updt_s:0.080 data_s:0.215
+INFO 2025-08-06 11:26:43 ts/train.py:232 step:19K smpl:150K ep:376 epch:7.52 loss:0.024 grdn:0.378 lr:9.2e-05 updt_s:0.080 data_s:0.216
+INFO 2025-08-06 11:27:42 ts/train.py:232 step:19K smpl:152K ep:380 epch:7.60 loss:0.023 grdn:0.411 lr:9.2e-05 updt_s:0.079 data_s:0.214
+INFO 2025-08-06 11:28:41 ts/train.py:232 step:19K smpl:154K ep:384 epch:7.68 loss:0.022 grdn:0.374 lr:9.2e-05 updt_s:0.080 data_s:0.215
+INFO 2025-08-06 11:29:41 ts/train.py:232 step:19K smpl:155K ep:388 epch:7.76 loss:0.022 grdn:0.383 lr:9.1e-05 updt_s:0.077 data_s:0.221
+INFO 2025-08-06 11:30:40 ts/train.py:232 step:20K smpl:157K ep:392 epch:7.84 loss:0.021 grdn:0.371 lr:9.1e-05 updt_s:0.081 data_s:0.214
+INFO 2025-08-06 11:31:39 ts/train.py:232 step:20K smpl:158K ep:396 epch:7.92 loss:0.022 grdn:0.376 lr:9.1e-05 updt_s:0.063 data_s:0.232
+INFO 2025-08-06 11:32:38 ts/train.py:232 step:20K smpl:160K ep:400 epch:8.00 loss:0.024 grdn:0.393 lr:9.1e-05 updt_s:0.080 data_s:0.213
+INFO 2025-08-06 11:32:38 ts/train.py:241 Checkpoint policy after step 20000
+INFO 2025-08-06 11:33:45 ts/train.py:232 step:20K smpl:162K ep:404 epch:8.08 loss:0.023 grdn:0.395 lr:9.1e-05 updt_s:0.059 data_s:0.245
+INFO 2025-08-06 11:34:44 ts/train.py:232 step:20K smpl:163K ep:408 epch:8.16 loss:0.023 grdn:0.378 lr:9.1e-05 updt_s:0.079 data_s:0.212
+INFO 2025-08-06 11:35:42 ts/train.py:232 step:21K smpl:165K ep:412 epch:8.24 loss:0.022 grdn:0.378 lr:9.0e-05 updt_s:0.080 data_s:0.211
+INFO 2025-08-06 11:36:40 ts/train.py:232 step:21K smpl:166K ep:416 epch:8.32 loss:0.022 grdn:0.378 lr:9.0e-05 updt_s:0.079 data_s:0.212
+INFO 2025-08-06 11:37:39 ts/train.py:232 step:21K smpl:168K ep:420 epch:8.40 loss:0.024 grdn:0.398 lr:9.0e-05 updt_s:0.080 data_s:0.211
+INFO 2025-08-06 11:38:37 ts/train.py:232 step:21K smpl:170K ep:424 epch:8.48 loss:0.022 grdn:0.372 lr:9.0e-05 updt_s:0.080 data_s:0.211
+INFO 2025-08-06 11:39:36 ts/train.py:232 step:21K smpl:171K ep:428 epch:8.56 loss:0.022 grdn:0.376 lr:9.0e-05 updt_s:0.080 data_s:0.212
+INFO 2025-08-06 11:40:34 ts/train.py:232 step:22K smpl:173K ep:432 epch:8.64 loss:0.022 grdn:0.364 lr:8.9e-05 updt_s:0.080 data_s:0.212
+INFO 2025-08-06 11:41:32 ts/train.py:232 step:22K smpl:174K ep:436 epch:8.72 loss:0.023 grdn:0.371 lr:8.9e-05 updt_s:0.080 data_s:0.211
+INFO 2025-08-06 11:42:31 ts/train.py:232 step:22K smpl:176K ep:440 epch:8.80 loss:0.023 grdn:0.380 lr:8.9e-05 updt_s:0.082 data_s:0.210
+INFO 2025-08-06 11:43:30 ts/train.py:232 step:22K smpl:178K ep:444 epch:8.88 loss:0.022 grdn:0.371 lr:8.9e-05 updt_s:0.070 data_s:0.225
+INFO 2025-08-06 11:44:30 ts/train.py:232 step:22K smpl:179K ep:448 epch:8.96 loss:0.023 grdn:0.395 lr:8.9e-05 updt_s:0.061 data_s:0.237
+INFO 2025-08-06 11:45:29 ts/train.py:232 step:23K smpl:181K ep:452 epch:9.04 loss:0.020 grdn:0.371 lr:8.8e-05 updt_s:0.072 data_s:0.225
+INFO 2025-08-06 11:46:29 ts/train.py:232 step:23K smpl:182K ep:456 epch:9.12 loss:0.021 grdn:0.364 lr:8.8e-05 updt_s:0.081 data_s:0.215
+INFO 2025-08-06 11:47:28 ts/train.py:232 step:23K smpl:184K ep:460 epch:9.20 loss:0.022 grdn:0.356 lr:8.8e-05 updt_s:0.079 data_s:0.215
+INFO 2025-08-06 11:48:27 ts/train.py:232 step:23K smpl:186K ep:464 epch:9.28 loss:0.020 grdn:0.345 lr:8.8e-05 updt_s:0.080 data_s:0.217
+INFO 2025-08-06 11:49:27 ts/train.py:232 step:23K smpl:187K ep:468 epch:9.36 loss:0.022 grdn:0.366 lr:8.8e-05 updt_s:0.082 data_s:0.214
+INFO 2025-08-06 11:50:26 ts/train.py:232 step:24K smpl:189K ep:472 epch:9.44 loss:0.021 grdn:0.364 lr:8.7e-05 updt_s:0.081 data_s:0.216
+INFO 2025-08-06 11:51:25 ts/train.py:232 step:24K smpl:190K ep:476 epch:9.52 loss:0.020 grdn:0.363 lr:8.7e-05 updt_s:0.081 data_s:0.214
+INFO 2025-08-06 11:52:25 ts/train.py:232 step:24K smpl:192K ep:480 epch:9.60 loss:0.022 grdn:0.375 lr:8.7e-05 updt_s:0.082 data_s:0.213
+INFO 2025-08-06 11:53:24 ts/train.py:232 step:24K smpl:194K ep:484 epch:9.68 loss:0.021 grdn:0.373 lr:8.7e-05 updt_s:0.082 data_s:0.214
+INFO 2025-08-06 11:54:23 ts/train.py:232 step:24K smpl:195K ep:488 epch:9.76 loss:0.022 grdn:0.352 lr:8.7e-05 updt_s:0.083 data_s:0.213
+INFO 2025-08-06 11:55:23 ts/train.py:232 step:25K smpl:197K ep:492 epch:9.84 loss:0.018 grdn:0.337 lr:8.6e-05 updt_s:0.078 data_s:0.223
+INFO 2025-08-06 11:56:23 ts/train.py:232 step:25K smpl:198K ep:496 epch:9.92 loss:0.020 grdn:0.353 lr:8.6e-05 updt_s:0.059 data_s:0.240
+INFO 2025-08-06 11:57:23 ts/train.py:232 step:25K smpl:200K ep:500 epch:10.00 loss:0.020 grdn:0.371 lr:8.6e-05 updt_s:0.067 data_s:0.230
+INFO 2025-08-06 11:58:22 ts/train.py:232 step:25K smpl:202K ep:504 epch:10.08 loss:0.021 grdn:0.355 lr:8.6e-05 updt_s:0.070 data_s:0.226
+INFO 2025-08-06 11:59:22 ts/train.py:232 step:25K smpl:203K ep:508 epch:10.16 loss:0.022 grdn:0.377 lr:8.5e-05 updt_s:0.080 data_s:0.217
+INFO 2025-08-06 12:00:22 ts/train.py:232 step:26K smpl:205K ep:512 epch:10.24 loss:0.022 grdn:0.363 lr:8.5e-05 updt_s:0.083 data_s:0.215
+INFO 2025-08-06 12:01:21 ts/train.py:232 step:26K smpl:206K ep:516 epch:10.32 loss:0.020 grdn:0.361 lr:8.5e-05 updt_s:0.080 data_s:0.219
+INFO 2025-08-06 12:02:21 ts/train.py:232 step:26K smpl:208K ep:520 epch:10.40 loss:0.019 grdn:0.335 lr:8.5e-05 updt_s:0.082 data_s:0.215
+INFO 2025-08-06 12:03:20 ts/train.py:232 step:26K smpl:210K ep:524 epch:10.48 loss:0.019 grdn:0.353 lr:8.5e-05 updt_s:0.080 data_s:0.216
+INFO 2025-08-06 12:04:20 ts/train.py:232 step:26K smpl:211K ep:528 epch:10.56 loss:0.020 grdn:0.351 lr:8.4e-05 updt_s:0.083 data_s:0.215
+INFO 2025-08-06 12:05:20 ts/train.py:232 step:27K smpl:213K ep:532 epch:10.64 loss:0.021 grdn:0.371 lr:8.4e-05 updt_s:0.080 data_s:0.217
+INFO 2025-08-06 12:06:19 ts/train.py:232 step:27K smpl:214K ep:536 epch:10.72 loss:0.020 grdn:0.370 lr:8.4e-05 updt_s:0.081 data_s:0.215
+INFO 2025-08-06 12:07:18 ts/train.py:232 step:27K smpl:216K ep:540 epch:10.80 loss:0.019 grdn:0.344 lr:8.4e-05 updt_s:0.083 data_s:0.213
+INFO 2025-08-06 12:08:18 ts/train.py:232 step:27K smpl:218K ep:544 epch:10.88 loss:0.021 grdn:0.384 lr:8.3e-05 updt_s:0.056 data_s:0.239
+INFO 2025-08-06 12:09:18 ts/train.py:232 step:27K smpl:219K ep:548 epch:10.96 loss:0.020 grdn:0.340 lr:8.3e-05 updt_s:0.078 data_s:0.221
+INFO 2025-08-06 12:10:17 ts/train.py:232 step:28K smpl:221K ep:552 epch:11.04 loss:0.020 grdn:0.358 lr:8.3e-05 updt_s:0.080 data_s:0.218
+INFO 2025-08-06 12:11:17 ts/train.py:232 step:28K smpl:222K ep:556 epch:11.12 loss:0.022 grdn:0.394 lr:8.3e-05 updt_s:0.079 data_s:0.218
+INFO 2025-08-06 12:12:16 ts/train.py:232 step:28K smpl:224K ep:560 epch:11.20 loss:0.020 grdn:0.370 lr:8.2e-05 updt_s:0.078 data_s:0.219
+INFO 2025-08-06 12:13:16 ts/train.py:232 step:28K smpl:226K ep:564 epch:11.28 loss:0.021 grdn:0.361 lr:8.2e-05 updt_s:0.080 data_s:0.217
+INFO 2025-08-06 12:14:16 ts/train.py:232 step:28K smpl:227K ep:568 epch:11.36 loss:0.020 grdn:0.350 lr:8.2e-05 updt_s:0.079 data_s:0.218
+INFO 2025-08-06 12:15:15 ts/train.py:232 step:29K smpl:229K ep:572 epch:11.44 loss:0.022 grdn:0.357 lr:8.2e-05 updt_s:0.079 data_s:0.217
+INFO 2025-08-06 12:16:15 ts/train.py:232 step:29K smpl:230K ep:576 epch:11.52 loss:0.020 grdn:0.357 lr:8.1e-05 updt_s:0.081 data_s:0.216
+INFO 2025-08-06 12:17:15 ts/train.py:232 step:29K smpl:232K ep:580 epch:11.60 loss:0.021 grdn:0.366 lr:8.1e-05 updt_s:0.076 data_s:0.224
+INFO 2025-08-06 12:18:14 ts/train.py:232 step:29K smpl:234K ep:584 epch:11.68 loss:0.022 grdn:0.369 lr:8.1e-05 updt_s:0.079 data_s:0.218
+INFO 2025-08-06 12:19:14 ts/train.py:232 step:29K smpl:235K ep:588 epch:11.76 loss:0.019 grdn:0.349 lr:8.1e-05 updt_s:0.081 data_s:0.216
+INFO 2025-08-06 12:20:15 ts/train.py:232 step:30K smpl:237K ep:592 epch:11.84 loss:0.020 grdn:0.368 lr:8.0e-05 updt_s:0.070 data_s:0.234
+INFO 2025-08-06 12:21:14 ts/train.py:232 step:30K smpl:238K ep:596 epch:11.92 loss:0.018 grdn:0.344 lr:8.0e-05 updt_s:0.063 data_s:0.232
+INFO 2025-08-06 12:22:13 ts/train.py:232 step:30K smpl:240K ep:600 epch:12.00 loss:0.019 grdn:0.346 lr:8.0e-05 updt_s:0.054 data_s:0.241
+INFO 2025-08-06 12:23:13 ts/train.py:232 step:30K smpl:242K ep:604 epch:12.08 loss:0.020 grdn:0.346 lr:8.0e-05 updt_s:0.062 data_s:0.236
+INFO 2025-08-06 12:24:12 ts/train.py:232 step:30K smpl:243K ep:608 epch:12.16 loss:0.018 grdn:0.349 lr:7.9e-05 updt_s:0.055 data_s:0.241
+INFO 2025-08-06 12:25:11 ts/train.py:232 step:31K smpl:245K ep:612 epch:12.24 loss:0.017 grdn:0.336 lr:7.9e-05 updt_s:0.060 data_s:0.236
+INFO 2025-08-06 12:26:10 ts/train.py:232 step:31K smpl:246K ep:616 epch:12.32 loss:0.018 grdn:0.338 lr:7.9e-05 updt_s:0.060 data_s:0.235
+INFO 2025-08-06 12:27:09 ts/train.py:232 step:31K smpl:248K ep:620 epch:12.40 loss:0.018 grdn:0.332 lr:7.9e-05 updt_s:0.063 data_s:0.232
+INFO 2025-08-06 12:28:09 ts/train.py:232 step:31K smpl:250K ep:624 epch:12.48 loss:0.020 grdn:0.366 lr:7.8e-05 updt_s:0.057 data_s:0.239
+INFO 2025-08-06 12:29:08 ts/train.py:232 step:31K smpl:251K ep:628 epch:12.56 loss:0.019 grdn:0.360 lr:7.8e-05 updt_s:0.070 data_s:0.226
+INFO 2025-08-06 12:30:07 ts/train.py:232 step:32K smpl:253K ep:632 epch:12.64 loss:0.018 grdn:0.340 lr:7.8e-05 updt_s:0.075 data_s:0.221
+INFO 2025-08-06 12:31:06 ts/train.py:232 step:32K smpl:254K ep:636 epch:12.72 loss:0.019 grdn:0.344 lr:7.8e-05 updt_s:0.063 data_s:0.230
+INFO 2025-08-06 12:32:05 ts/train.py:232 step:32K smpl:256K ep:640 epch:12.80 loss:0.019 grdn:0.339 lr:7.7e-05 updt_s:0.054 data_s:0.240
+INFO 2025-08-06 12:33:04 ts/train.py:232 step:32K smpl:258K ep:644 epch:12.88 loss:0.019 grdn:0.361 lr:7.7e-05 updt_s:0.070 data_s:0.224
+INFO 2025-08-06 12:34:02 ts/train.py:232 step:32K smpl:259K ep:648 epch:12.96 loss:0.020 grdn:0.366 lr:7.7e-05 updt_s:0.082 data_s:0.207
+INFO 2025-08-06 12:35:00 ts/train.py:232 step:33K smpl:261K ep:652 epch:13.04 loss:0.020 grdn:0.372 lr:7.7e-05 updt_s:0.080 data_s:0.208
+INFO 2025-08-06 12:35:58 ts/train.py:232 step:33K smpl:262K ep:656 epch:13.12 loss:0.018 grdn:0.341 lr:7.6e-05 updt_s:0.081 data_s:0.209
+INFO 2025-08-06 12:36:56 ts/train.py:232 step:33K smpl:264K ep:660 epch:13.20 loss:0.018 grdn:0.339 lr:7.6e-05 updt_s:0.080 data_s:0.210
+INFO 2025-08-06 12:37:54 ts/train.py:232 step:33K smpl:266K ep:664 epch:13.28 loss:0.020 grdn:0.369 lr:7.6e-05 updt_s:0.081 data_s:0.210
+INFO 2025-08-06 12:38:52 ts/train.py:232 step:33K smpl:267K ep:668 epch:13.36 loss:0.018 grdn:0.332 lr:7.5e-05 updt_s:0.079 data_s:0.209
+INFO 2025-08-06 12:39:50 ts/train.py:232 step:34K smpl:269K ep:672 epch:13.44 loss:0.020 grdn:0.362 lr:7.5e-05 updt_s:0.081 data_s:0.208
+INFO 2025-08-06 12:40:48 ts/train.py:232 step:34K smpl:270K ep:676 epch:13.52 loss:0.020 grdn:0.343 lr:7.5e-05 updt_s:0.081 data_s:0.207
+INFO 2025-08-06 12:41:46 ts/train.py:232 step:34K smpl:272K ep:680 epch:13.60 loss:0.018 grdn:0.353 lr:7.5e-05 updt_s:0.080 data_s:0.209
+INFO 2025-08-06 12:42:44 ts/train.py:232 step:34K smpl:274K ep:684 epch:13.68 loss:0.018 grdn:0.341 lr:7.4e-05 updt_s:0.080 data_s:0.209
+INFO 2025-08-06 12:43:43 ts/train.py:232 step:34K smpl:275K ep:688 epch:13.76 loss:0.019 grdn:0.354 lr:7.4e-05 updt_s:0.078 data_s:0.219
+INFO 2025-08-06 12:44:43 ts/train.py:232 step:35K smpl:277K ep:692 epch:13.84 loss:0.019 grdn:0.361 lr:7.4e-05 updt_s:0.054 data_s:0.244
+INFO 2025-08-06 12:45:42 ts/train.py:232 step:35K smpl:278K ep:696 epch:13.92 loss:0.021 grdn:0.349 lr:7.4e-05 updt_s:0.056 data_s:0.240
+INFO 2025-08-06 12:46:42 ts/train.py:232 step:35K smpl:280K ep:700 epch:14.00 loss:0.018 grdn:0.334 lr:7.3e-05 updt_s:0.051 data_s:0.248
+INFO 2025-08-06 12:47:42 ts/train.py:232 step:35K smpl:282K ep:704 epch:14.08 loss:0.017 grdn:0.334 lr:7.3e-05 updt_s:0.045 data_s:0.253
+INFO 2025-08-06 12:48:42 ts/train.py:232 step:35K smpl:283K ep:708 epch:14.16 loss:0.019 grdn:0.341 lr:7.3e-05 updt_s:0.043 data_s:0.255
+INFO 2025-08-06 12:49:42 ts/train.py:232 step:36K smpl:285K ep:712 epch:14.24 loss:0.016 grdn:0.322 lr:7.2e-05 updt_s:0.058 data_s:0.240
+INFO 2025-08-06 12:50:42 ts/train.py:232 step:36K smpl:286K ep:716 epch:14.32 loss:0.017 grdn:0.339 lr:7.2e-05 updt_s:0.065 data_s:0.236
+INFO 2025-08-06 12:51:41 ts/train.py:232 step:36K smpl:288K ep:720 epch:14.40 loss:0.018 grdn:0.332 lr:7.2e-05 updt_s:0.066 data_s:0.231
+INFO 2025-08-06 12:52:41 ts/train.py:232 step:36K smpl:290K ep:724 epch:14.48 loss:0.019 grdn:0.352 lr:7.2e-05 updt_s:0.062 data_s:0.237
+INFO 2025-08-06 12:53:41 ts/train.py:232 step:36K smpl:291K ep:728 epch:14.56 loss:0.017 grdn:0.321 lr:7.1e-05 updt_s:0.062 data_s:0.236
+INFO 2025-08-06 12:54:41 ts/train.py:232 step:37K smpl:293K ep:732 epch:14.64 loss:0.017 grdn:0.348 lr:7.1e-05 updt_s:0.073 data_s:0.226
+INFO 2025-08-06 12:55:40 ts/train.py:232 step:37K smpl:294K ep:736 epch:14.72 loss:0.019 grdn:0.343 lr:7.1e-05 updt_s:0.070 data_s:0.228
+INFO 2025-08-06 12:56:40 ts/train.py:232 step:37K smpl:296K ep:740 epch:14.80 loss:0.018 grdn:0.356 lr:7.0e-05 updt_s:0.069 data_s:0.227
+INFO 2025-08-06 12:57:38 ts/train.py:232 step:37K smpl:298K ep:744 epch:14.88 loss:0.018 grdn:0.363 lr:7.0e-05 updt_s:0.076 data_s:0.213
+INFO 2025-08-06 12:58:35 ts/train.py:232 step:37K smpl:299K ep:748 epch:14.96 loss:0.017 grdn:0.350 lr:7.0e-05 updt_s:0.069 data_s:0.219
+INFO 2025-08-06 12:59:33 ts/train.py:232 step:38K smpl:301K ep:752 epch:15.04 loss:0.016 grdn:0.319 lr:7.0e-05 updt_s:0.064 data_s:0.222
+INFO 2025-08-06 13:00:31 ts/train.py:232 step:38K smpl:302K ep:756 epch:15.12 loss:0.016 grdn:0.314 lr:6.9e-05 updt_s:0.066 data_s:0.223
+INFO 2025-08-06 13:01:28 ts/train.py:232 step:38K smpl:304K ep:760 epch:15.20 loss:0.017 grdn:0.339 lr:6.9e-05 updt_s:0.071 data_s:0.215
+INFO 2025-08-06 13:02:25 ts/train.py:232 step:38K smpl:306K ep:764 epch:15.28 loss:0.018 grdn:0.340 lr:6.9e-05 updt_s:0.077 data_s:0.210
+INFO 2025-08-06 13:03:23 ts/train.py:232 step:38K smpl:307K ep:768 epch:15.36 loss:0.017 grdn:0.342 lr:6.8e-05 updt_s:0.081 data_s:0.206
+INFO 2025-08-06 13:04:21 ts/train.py:232 step:39K smpl:309K ep:772 epch:15.44 loss:0.018 grdn:0.355 lr:6.8e-05 updt_s:0.080 data_s:0.209
+INFO 2025-08-06 13:05:19 ts/train.py:232 step:39K smpl:310K ep:776 epch:15.52 loss:0.018 grdn:0.334 lr:6.8e-05 updt_s:0.081 data_s:0.207
+INFO 2025-08-06 13:06:16 ts/train.py:232 step:39K smpl:312K ep:780 epch:15.60 loss:0.019 grdn:0.354 lr:6.8e-05 updt_s:0.082 data_s:0.205
+INFO 2025-08-06 13:07:13 ts/train.py:232 step:39K smpl:314K ep:784 epch:15.68 loss:0.018 grdn:0.348 lr:6.7e-05 updt_s:0.080 data_s:0.206
+INFO 2025-08-06 13:08:12 ts/train.py:232 step:39K smpl:315K ep:788 epch:15.76 loss:0.020 grdn:0.383 lr:6.7e-05 updt_s:0.069 data_s:0.225
+INFO 2025-08-06 13:09:11 ts/train.py:232 step:40K smpl:317K ep:792 epch:15.84 loss:0.016 grdn:0.333 lr:6.7e-05 updt_s:0.072 data_s:0.222
+INFO 2025-08-06 13:10:11 ts/train.py:232 step:40K smpl:318K ep:796 epch:15.92 loss:0.017 grdn:0.333 lr:6.6e-05 updt_s:0.074 data_s:0.223
+INFO 2025-08-06 13:11:10 ts/train.py:232 step:40K smpl:320K ep:800 epch:16.00 loss:0.018 grdn:0.352 lr:6.6e-05 updt_s:0.072 data_s:0.224
+INFO 2025-08-06 13:11:10 ts/train.py:241 Checkpoint policy after step 40000
+INFO 2025-08-06 13:12:17 ts/train.py:232 step:40K smpl:322K ep:804 epch:16.08 loss:0.018 grdn:0.363 lr:6.6e-05 updt_s:0.072 data_s:0.232
+INFO 2025-08-06 13:13:16 ts/train.py:232 step:40K smpl:323K ep:808 epch:16.16 loss:0.018 grdn:0.347 lr:6.5e-05 updt_s:0.074 data_s:0.220
+INFO 2025-08-06 13:14:15 ts/train.py:232 step:41K smpl:325K ep:812 epch:16.24 loss:0.016 grdn:0.333 lr:6.5e-05 updt_s:0.071 data_s:0.225
+INFO 2025-08-06 13:15:14 ts/train.py:232 step:41K smpl:326K ep:816 epch:16.32 loss:0.017 grdn:0.330 lr:6.5e-05 updt_s:0.065 data_s:0.230
+INFO 2025-08-06 13:16:13 ts/train.py:232 step:41K smpl:328K ep:820 epch:16.40 loss:0.018 grdn:0.348 lr:6.5e-05 updt_s:0.064 data_s:0.231
+INFO 2025-08-06 13:17:12 ts/train.py:232 step:41K smpl:330K ep:824 epch:16.48 loss:0.018 grdn:0.347 lr:6.4e-05 updt_s:0.063 data_s:0.232
+INFO 2025-08-06 13:18:12 ts/train.py:232 step:41K smpl:331K ep:828 epch:16.56 loss:0.015 grdn:0.321 lr:6.4e-05 updt_s:0.068 data_s:0.229
+INFO 2025-08-06 13:19:11 ts/train.py:232 step:42K smpl:333K ep:832 epch:16.64 loss:0.018 grdn:0.343 lr:6.4e-05 updt_s:0.078 data_s:0.218
+INFO 2025-08-06 13:20:12 ts/train.py:232 step:42K smpl:334K ep:836 epch:16.72 loss:0.017 grdn:0.349 lr:6.3e-05 updt_s:0.078 data_s:0.226
+INFO 2025-08-06 13:21:12 ts/train.py:232 step:42K smpl:336K ep:840 epch:16.80 loss:0.017 grdn:0.347 lr:6.3e-05 updt_s:0.067 data_s:0.230
+INFO 2025-08-06 13:22:10 ts/train.py:232 step:42K smpl:338K ep:844 epch:16.88 loss:0.016 grdn:0.308 lr:6.3e-05 updt_s:0.080 data_s:0.214
+INFO 2025-08-06 13:23:09 ts/train.py:232 step:42K smpl:339K ep:848 epch:16.96 loss:0.018 grdn:0.349 lr:6.2e-05 updt_s:0.080 data_s:0.213
+INFO 2025-08-06 13:24:08 ts/train.py:232 step:43K smpl:341K ep:852 epch:17.04 loss:0.018 grdn:0.352 lr:6.2e-05 updt_s:0.082 data_s:0.211
+INFO 2025-08-06 13:25:07 ts/train.py:232 step:43K smpl:342K ep:856 epch:17.12 loss:0.018 grdn:0.360 lr:6.2e-05 updt_s:0.081 data_s:0.213
+INFO 2025-08-06 13:26:05 ts/train.py:232 step:43K smpl:344K ep:860 epch:17.20 loss:0.018 grdn:0.339 lr:6.1e-05 updt_s:0.081 data_s:0.212
+INFO 2025-08-06 13:27:04 ts/train.py:232 step:43K smpl:346K ep:864 epch:17.28 loss:0.017 grdn:0.324 lr:6.1e-05 updt_s:0.080 data_s:0.213
+INFO 2025-08-06 13:28:03 ts/train.py:232 step:43K smpl:347K ep:868 epch:17.36 loss:0.016 grdn:0.315 lr:6.1e-05 updt_s:0.081 data_s:0.212
+INFO 2025-08-06 13:29:01 ts/train.py:232 step:44K smpl:349K ep:872 epch:17.44 loss:0.015 grdn:0.319 lr:6.1e-05 updt_s:0.081 data_s:0.211
+INFO 2025-08-06 13:30:00 ts/train.py:232 step:44K smpl:350K ep:876 epch:17.52 loss:0.016 grdn:0.336 lr:6.0e-05 updt_s:0.081 data_s:0.212
+INFO 2025-08-06 13:30:59 ts/train.py:232 step:44K smpl:352K ep:880 epch:17.60 loss:0.016 grdn:0.342 lr:6.0e-05 updt_s:0.080 data_s:0.212
+INFO 2025-08-06 13:31:57 ts/train.py:232 step:44K smpl:354K ep:884 epch:17.68 loss:0.016 grdn:0.347 lr:6.0e-05 updt_s:0.081 data_s:0.210
+INFO 2025-08-06 13:32:58 ts/train.py:232 step:44K smpl:355K ep:888 epch:17.76 loss:0.017 grdn:0.341 lr:5.9e-05 updt_s:0.051 data_s:0.255
+INFO 2025-08-06 13:33:58 ts/train.py:232 step:45K smpl:357K ep:892 epch:17.84 loss:0.016 grdn:0.323 lr:5.9e-05 updt_s:0.051 data_s:0.246
+INFO 2025-08-06 13:34:58 ts/train.py:232 step:45K smpl:358K ep:896 epch:17.92 loss:0.017 grdn:0.358 lr:5.9e-05 updt_s:0.047 data_s:0.252
+INFO 2025-08-06 13:35:57 ts/train.py:232 step:45K smpl:360K ep:900 epch:18.00 loss:0.020 grdn:0.374 lr:5.8e-05 updt_s:0.053 data_s:0.245
+INFO 2025-08-06 13:36:57 ts/train.py:232 step:45K smpl:362K ep:904 epch:18.08 loss:0.017 grdn:0.347 lr:5.8e-05 updt_s:0.066 data_s:0.231
+INFO 2025-08-06 13:37:56 ts/train.py:232 step:45K smpl:363K ep:908 epch:18.16 loss:0.016 grdn:0.317 lr:5.8e-05 updt_s:0.071 data_s:0.223
+INFO 2025-08-06 13:38:56 ts/train.py:232 step:46K smpl:365K ep:912 epch:18.24 loss:0.017 grdn:0.340 lr:5.7e-05 updt_s:0.083 data_s:0.215
+INFO 2025-08-06 13:39:55 ts/train.py:232 step:46K smpl:366K ep:916 epch:18.32 loss:0.018 grdn:0.366 lr:5.7e-05 updt_s:0.084 data_s:0.213
+INFO 2025-08-06 13:40:55 ts/train.py:232 step:46K smpl:368K ep:920 epch:18.40 loss:0.017 grdn:0.327 lr:5.7e-05 updt_s:0.082 data_s:0.215
+INFO 2025-08-06 13:41:55 ts/train.py:232 step:46K smpl:370K ep:924 epch:18.48 loss:0.017 grdn:0.317 lr:5.7e-05 updt_s:0.083 data_s:0.215
+INFO 2025-08-06 13:42:54 ts/train.py:232 step:46K smpl:371K ep:928 epch:18.56 loss:0.016 grdn:0.313 lr:5.6e-05 updt_s:0.082 data_s:0.214
+INFO 2025-08-06 13:43:53 ts/train.py:232 step:47K smpl:373K ep:932 epch:18.64 loss:0.016 grdn:0.327 lr:5.6e-05 updt_s:0.082 data_s:0.214
+INFO 2025-08-06 13:44:54 ts/train.py:232 step:47K smpl:374K ep:936 epch:18.72 loss:0.017 grdn:0.321 lr:5.6e-05 updt_s:0.076 data_s:0.229
+INFO 2025-08-06 13:45:53 ts/train.py:232 step:47K smpl:376K ep:940 epch:18.80 loss:0.016 grdn:0.321 lr:5.5e-05 updt_s:0.079 data_s:0.216
+INFO 2025-08-06 13:46:52 ts/train.py:232 step:47K smpl:378K ep:944 epch:18.88 loss:0.016 grdn:0.322 lr:5.5e-05 updt_s:0.080 data_s:0.213
+INFO 2025-08-06 13:47:51 ts/train.py:232 step:47K smpl:379K ep:948 epch:18.96 loss:0.015 grdn:0.328 lr:5.5e-05 updt_s:0.079 data_s:0.215
+INFO 2025-08-06 13:48:50 ts/train.py:232 step:48K smpl:381K ep:952 epch:19.04 loss:0.016 grdn:0.323 lr:5.4e-05 updt_s:0.079 data_s:0.216
+INFO 2025-08-06 13:49:49 ts/train.py:232 step:48K smpl:382K ep:956 epch:19.12 loss:0.016 grdn:0.315 lr:5.4e-05 updt_s:0.079 data_s:0.214
+INFO 2025-08-06 13:50:48 ts/train.py:232 step:48K smpl:384K ep:960 epch:19.20 loss:0.016 grdn:0.316 lr:5.4e-05 updt_s:0.079 data_s:0.215
+INFO 2025-08-06 13:51:47 ts/train.py:232 step:48K smpl:386K ep:964 epch:19.28 loss:0.016 grdn:0.328 lr:5.3e-05 updt_s:0.079 data_s:0.216
+INFO 2025-08-06 13:52:46 ts/train.py:232 step:48K smpl:387K ep:968 epch:19.36 loss:0.015 grdn:0.332 lr:5.3e-05 updt_s:0.078 data_s:0.215
+INFO 2025-08-06 13:53:45 ts/train.py:232 step:49K smpl:389K ep:972 epch:19.44 loss:0.018 grdn:0.334 lr:5.3e-05 updt_s:0.079 data_s:0.217
+INFO 2025-08-06 13:54:44 ts/train.py:232 step:49K smpl:390K ep:976 epch:19.52 loss:0.016 grdn:0.327 lr:5.2e-05 updt_s:0.078 data_s:0.216
+INFO 2025-08-06 13:55:43 ts/train.py:232 step:49K smpl:392K ep:980 epch:19.60 loss:0.017 grdn:0.346 lr:5.2e-05 updt_s:0.079 data_s:0.216
+INFO 2025-08-06 13:56:42 ts/train.py:232 step:49K smpl:394K ep:984 epch:19.68 loss:0.016 grdn:0.329 lr:5.2e-05 updt_s:0.074 data_s:0.220
+INFO 2025-08-06 13:57:43 ts/train.py:232 step:49K smpl:395K ep:988 epch:19.76 loss:0.016 grdn:0.326 lr:5.1e-05 updt_s:0.073 data_s:0.231
+INFO 2025-08-06 13:58:43 ts/train.py:232 step:50K smpl:397K ep:992 epch:19.84 loss:0.016 grdn:0.309 lr:5.1e-05 updt_s:0.069 data_s:0.228
+INFO 2025-08-06 13:59:42 ts/train.py:232 step:50K smpl:398K ep:996 epch:19.92 loss:0.015 grdn:0.323 lr:5.1e-05 updt_s:0.069 data_s:0.228
+INFO 2025-08-06 14:00:42 ts/train.py:232 step:50K smpl:400K ep:1K epch:20.00 loss:0.016 grdn:0.334 lr:5.1e-05 updt_s:0.071 data_s:0.225
+INFO 2025-08-06 14:01:41 ts/train.py:232 step:50K smpl:402K ep:1K epch:20.08 loss:0.017 grdn:0.336 lr:5.0e-05 updt_s:0.075 data_s:0.222
+INFO 2025-08-06 14:02:40 ts/train.py:232 step:50K smpl:403K ep:1K epch:20.16 loss:0.015 grdn:0.312 lr:5.0e-05 updt_s:0.076 data_s:0.219
+INFO 2025-08-06 14:03:39 ts/train.py:232 step:51K smpl:405K ep:1K epch:20.24 loss:0.015 grdn:0.330 lr:5.0e-05 updt_s:0.073 data_s:0.222
+INFO 2025-08-06 14:04:39 ts/train.py:232 step:51K smpl:406K ep:1K epch:20.32 loss:0.014 grdn:0.296 lr:4.9e-05 updt_s:0.069 data_s:0.228
+INFO 2025-08-06 14:05:38 ts/train.py:232 step:51K smpl:408K ep:1K epch:20.40 loss:0.017 grdn:0.343 lr:4.9e-05 updt_s:0.068 data_s:0.226
+INFO 2025-08-06 14:06:37 ts/train.py:232 step:51K smpl:410K ep:1K epch:20.48 loss:0.014 grdn:0.318 lr:4.9e-05 updt_s:0.079 data_s:0.218
+INFO 2025-08-06 14:07:37 ts/train.py:232 step:51K smpl:411K ep:1K epch:20.56 loss:0.016 grdn:0.318 lr:4.8e-05 updt_s:0.074 data_s:0.222
+INFO 2025-08-06 14:08:36 ts/train.py:232 step:52K smpl:413K ep:1K epch:20.64 loss:0.016 grdn:0.331 lr:4.8e-05 updt_s:0.069 data_s:0.228
+INFO 2025-08-06 14:09:36 ts/train.py:232 step:52K smpl:414K ep:1K epch:20.72 loss:0.015 grdn:0.324 lr:4.8e-05 updt_s:0.070 data_s:0.231
+INFO 2025-08-06 14:10:36 ts/train.py:232 step:52K smpl:416K ep:1K epch:20.80 loss:0.017 grdn:0.333 lr:4.7e-05 updt_s:0.063 data_s:0.236
+INFO 2025-08-06 14:11:36 ts/train.py:232 step:52K smpl:418K ep:1K epch:20.88 loss:0.018 grdn:0.348 lr:4.7e-05 updt_s:0.052 data_s:0.247
+INFO 2025-08-06 14:12:37 ts/train.py:232 step:52K smpl:419K ep:1K epch:20.96 loss:0.016 grdn:0.330 lr:4.7e-05 updt_s:0.051 data_s:0.250
+INFO 2025-08-06 14:13:37 ts/train.py:232 step:53K smpl:421K ep:1K epch:21.04 loss:0.016 grdn:0.337 lr:4.6e-05 updt_s:0.062 data_s:0.238
+INFO 2025-08-06 14:14:37 ts/train.py:232 step:53K smpl:422K ep:1K epch:21.12 loss:0.015 grdn:0.322 lr:4.6e-05 updt_s:0.070 data_s:0.230
+INFO 2025-08-06 14:15:36 ts/train.py:232 step:53K smpl:424K ep:1K epch:21.20 loss:0.015 grdn:0.325 lr:4.6e-05 updt_s:0.062 data_s:0.235
+INFO 2025-08-06 14:16:37 ts/train.py:232 step:53K smpl:426K ep:1K epch:21.28 loss:0.015 grdn:0.309 lr:4.6e-05 updt_s:0.058 data_s:0.242
+INFO 2025-08-06 14:17:37 ts/train.py:232 step:53K smpl:427K ep:1K epch:21.36 loss:0.016 grdn:0.339 lr:4.5e-05 updt_s:0.060 data_s:0.239
+INFO 2025-08-06 14:18:37 ts/train.py:232 step:54K smpl:429K ep:1K epch:21.44 loss:0.015 grdn:0.326 lr:4.5e-05 updt_s:0.074 data_s:0.227
+INFO 2025-08-06 14:19:37 ts/train.py:232 step:54K smpl:430K ep:1K epch:21.52 loss:0.016 grdn:0.309 lr:4.5e-05 updt_s:0.072 data_s:0.227
+INFO 2025-08-06 14:20:37 ts/train.py:232 step:54K smpl:432K ep:1K epch:21.60 loss:0.016 grdn:0.332 lr:4.4e-05 updt_s:0.073 data_s:0.228
+INFO 2025-08-06 14:21:39 ts/train.py:232 step:54K smpl:434K ep:1K epch:21.68 loss:0.014 grdn:0.307 lr:4.4e-05 updt_s:0.058 data_s:0.251
+INFO 2025-08-06 14:22:41 ts/train.py:232 step:54K smpl:435K ep:1K epch:21.76 loss:0.014 grdn:0.300 lr:4.4e-05 updt_s:0.059 data_s:0.250
+INFO 2025-08-06 14:23:42 ts/train.py:232 step:55K smpl:437K ep:1K epch:21.84 loss:0.015 grdn:0.343 lr:4.3e-05 updt_s:0.072 data_s:0.233
+INFO 2025-08-06 14:24:44 ts/train.py:232 step:55K smpl:438K ep:1K epch:21.92 loss:0.015 grdn:0.333 lr:4.3e-05 updt_s:0.082 data_s:0.224
+INFO 2025-08-06 14:25:45 ts/train.py:232 step:55K smpl:440K ep:1K epch:22.00 loss:0.016 grdn:0.343 lr:4.3e-05 updt_s:0.083 data_s:0.223
+INFO 2025-08-06 14:26:46 ts/train.py:232 step:55K smpl:442K ep:1K epch:22.08 loss:0.015 grdn:0.335 lr:4.2e-05 updt_s:0.079 data_s:0.228
+INFO 2025-08-06 14:27:48 ts/train.py:232 step:55K smpl:443K ep:1K epch:22.16 loss:0.014 grdn:0.330 lr:4.2e-05 updt_s:0.082 data_s:0.224
+INFO 2025-08-06 14:28:49 ts/train.py:232 step:56K smpl:445K ep:1K epch:22.24 loss:0.017 grdn:0.333 lr:4.2e-05 updt_s:0.083 data_s:0.224
+INFO 2025-08-06 14:29:51 ts/train.py:232 step:56K smpl:446K ep:1K epch:22.32 loss:0.016 grdn:0.330 lr:4.1e-05 updt_s:0.079 data_s:0.228
+INFO 2025-08-06 14:30:52 ts/train.py:232 step:56K smpl:448K ep:1K epch:22.40 loss:0.016 grdn:0.329 lr:4.1e-05 updt_s:0.073 data_s:0.233
+INFO 2025-08-06 14:31:53 ts/train.py:232 step:56K smpl:450K ep:1K epch:22.48 loss:0.015 grdn:0.324 lr:4.1e-05 updt_s:0.071 data_s:0.234
+INFO 2025-08-06 14:32:55 ts/train.py:232 step:56K smpl:451K ep:1K epch:22.56 loss:0.014 grdn:0.314 lr:4.1e-05 updt_s:0.082 data_s:0.224
+INFO 2025-08-06 14:33:58 ts/train.py:232 step:57K smpl:453K ep:1K epch:22.64 loss:0.014 grdn:0.310 lr:4.0e-05 updt_s:0.073 data_s:0.241
+INFO 2025-08-06 14:34:59 ts/train.py:232 step:57K smpl:454K ep:1K epch:22.72 loss:0.015 grdn:0.315 lr:4.0e-05 updt_s:0.077 data_s:0.229
+INFO 2025-08-06 14:36:00 ts/train.py:232 step:57K smpl:456K ep:1K epch:22.80 loss:0.015 grdn:0.323 lr:4.0e-05 updt_s:0.074 data_s:0.232
+INFO 2025-08-06 14:37:01 ts/train.py:232 step:57K smpl:458K ep:1K epch:22.88 loss:0.015 grdn:0.311 lr:3.9e-05 updt_s:0.081 data_s:0.224
+INFO 2025-08-06 14:38:03 ts/train.py:232 step:57K smpl:459K ep:1K epch:22.96 loss:0.014 grdn:0.305 lr:3.9e-05 updt_s:0.080 data_s:0.226
+INFO 2025-08-06 14:39:04 ts/train.py:232 step:58K smpl:461K ep:1K epch:23.04 loss:0.015 grdn:0.324 lr:3.9e-05 updt_s:0.082 data_s:0.223
+INFO 2025-08-06 14:40:05 ts/train.py:232 step:58K smpl:462K ep:1K epch:23.12 loss:0.014 grdn:0.313 lr:3.8e-05 updt_s:0.082 data_s:0.222
+INFO 2025-08-06 14:41:06 ts/train.py:232 step:58K smpl:464K ep:1K epch:23.20 loss:0.015 grdn:0.347 lr:3.8e-05 updt_s:0.081 data_s:0.223
+INFO 2025-08-06 14:42:07 ts/train.py:232 step:58K smpl:466K ep:1K epch:23.28 loss:0.014 grdn:0.310 lr:3.8e-05 updt_s:0.081 data_s:0.224
+INFO 2025-08-06 14:43:06 ts/train.py:232 step:58K smpl:467K ep:1K epch:23.36 loss:0.015 grdn:0.308 lr:3.7e-05 updt_s:0.074 data_s:0.221
+INFO 2025-08-06 14:44:04 ts/train.py:232 step:59K smpl:469K ep:1K epch:23.44 loss:0.015 grdn:0.315 lr:3.7e-05 updt_s:0.073 data_s:0.216
+INFO 2025-08-06 14:45:02 ts/train.py:232 step:59K smpl:470K ep:1K epch:23.52 loss:0.014 grdn:0.315 lr:3.7e-05 updt_s:0.073 data_s:0.215
+INFO 2025-08-06 14:45:59 ts/train.py:232 step:59K smpl:472K ep:1K epch:23.60 loss:0.014 grdn:0.322 lr:3.7e-05 updt_s:0.068 data_s:0.219
+INFO 2025-08-06 14:46:56 ts/train.py:232 step:59K smpl:474K ep:1K epch:23.68 loss:0.014 grdn:0.305 lr:3.6e-05 updt_s:0.070 data_s:0.214
+INFO 2025-08-06 14:47:52 ts/train.py:232 step:59K smpl:475K ep:1K epch:23.76 loss:0.014 grdn:0.321 lr:3.6e-05 updt_s:0.071 data_s:0.211
+INFO 2025-08-06 14:48:49 ts/train.py:232 step:60K smpl:477K ep:1K epch:23.84 loss:0.014 grdn:0.310 lr:3.6e-05 updt_s:0.072 data_s:0.211
+INFO 2025-08-06 14:49:45 ts/train.py:232 step:60K smpl:478K ep:1K epch:23.92 loss:0.015 grdn:0.322 lr:3.5e-05 updt_s:0.070 data_s:0.209
+INFO 2025-08-06 14:50:41 ts/train.py:232 step:60K smpl:480K ep:1K epch:24.00 loss:0.014 grdn:0.316 lr:3.5e-05 updt_s:0.071 data_s:0.210
+INFO 2025-08-06 14:50:41 ts/train.py:241 Checkpoint policy after step 60000
+INFO 2025-08-06 14:51:44 ts/train.py:232 step:60K smpl:482K ep:1K epch:24.08 loss:0.015 grdn:0.330 lr:3.5e-05 updt_s:0.062 data_s:0.224
+INFO 2025-08-06 14:52:40 ts/train.py:232 step:60K smpl:483K ep:1K epch:24.16 loss:0.013 grdn:0.307 lr:3.4e-05 updt_s:0.063 data_s:0.214
+INFO 2025-08-06 14:53:36 ts/train.py:232 step:61K smpl:485K ep:1K epch:24.24 loss:0.014 grdn:0.312 lr:3.4e-05 updt_s:0.061 data_s:0.217
+INFO 2025-08-06 14:54:31 ts/train.py:232 step:61K smpl:486K ep:1K epch:24.32 loss:0.015 grdn:0.331 lr:3.4e-05 updt_s:0.064 data_s:0.214
+INFO 2025-08-06 14:55:27 ts/train.py:232 step:61K smpl:488K ep:1K epch:24.40 loss:0.013 grdn:0.292 lr:3.4e-05 updt_s:0.073 data_s:0.204
+INFO 2025-08-06 14:56:22 ts/train.py:232 step:61K smpl:490K ep:1K epch:24.48 loss:0.013 grdn:0.324 lr:3.3e-05 updt_s:0.066 data_s:0.211
+INFO 2025-08-06 14:57:18 ts/train.py:232 step:61K smpl:491K ep:1K epch:24.56 loss:0.015 grdn:0.322 lr:3.3e-05 updt_s:0.063 data_s:0.214
+INFO 2025-08-06 14:58:15 ts/train.py:232 step:62K smpl:493K ep:1K epch:24.64 loss:0.015 grdn:0.323 lr:3.3e-05 updt_s:0.058 data_s:0.228
+INFO 2025-08-06 14:59:12 ts/train.py:232 step:62K smpl:494K ep:1K epch:24.72 loss:0.015 grdn:0.310 lr:3.2e-05 updt_s:0.066 data_s:0.216
+INFO 2025-08-06 15:00:08 ts/train.py:232 step:62K smpl:496K ep:1K epch:24.80 loss:0.014 grdn:0.312 lr:3.2e-05 updt_s:0.056 data_s:0.226
+INFO 2025-08-06 15:01:05 ts/train.py:232 step:62K smpl:498K ep:1K epch:24.88 loss:0.013 grdn:0.315 lr:3.2e-05 updt_s:0.050 data_s:0.234
+INFO 2025-08-06 15:02:02 ts/train.py:232 step:62K smpl:499K ep:1K epch:24.96 loss:0.013 grdn:0.317 lr:3.1e-05 updt_s:0.051 data_s:0.230
+INFO 2025-08-06 15:02:58 ts/train.py:232 step:63K smpl:501K ep:1K epch:25.04 loss:0.013 grdn:0.297 lr:3.1e-05 updt_s:0.058 data_s:0.224
+INFO 2025-08-06 15:03:55 ts/train.py:232 step:63K smpl:502K ep:1K epch:25.12 loss:0.014 grdn:0.315 lr:3.1e-05 updt_s:0.066 data_s:0.216
+INFO 2025-08-06 15:04:51 ts/train.py:232 step:63K smpl:504K ep:1K epch:25.20 loss:0.013 grdn:0.308 lr:3.1e-05 updt_s:0.072 data_s:0.209
+INFO 2025-08-06 15:05:48 ts/train.py:232 step:63K smpl:506K ep:1K epch:25.28 loss:0.013 grdn:0.311 lr:3.0e-05 updt_s:0.072 data_s:0.210
+INFO 2025-08-06 15:06:44 ts/train.py:232 step:63K smpl:507K ep:1K epch:25.36 loss:0.014 grdn:0.303 lr:3.0e-05 updt_s:0.070 data_s:0.210
+INFO 2025-08-06 15:07:41 ts/train.py:232 step:64K smpl:509K ep:1K epch:25.44 loss:0.015 grdn:0.330 lr:3.0e-05 updt_s:0.071 data_s:0.212
+INFO 2025-08-06 15:08:37 ts/train.py:232 step:64K smpl:510K ep:1K epch:25.52 loss:0.014 grdn:0.325 lr:2.9e-05 updt_s:0.072 data_s:0.209
+INFO 2025-08-06 15:09:34 ts/train.py:232 step:64K smpl:512K ep:1K epch:25.60 loss:0.014 grdn:0.308 lr:2.9e-05 updt_s:0.063 data_s:0.222
+INFO 2025-08-06 15:10:30 ts/train.py:232 step:64K smpl:514K ep:1K epch:25.68 loss:0.014 grdn:0.304 lr:2.9e-05 updt_s:0.049 data_s:0.233
+INFO 2025-08-06 15:11:27 ts/train.py:232 step:64K smpl:515K ep:1K epch:25.76 loss:0.012 grdn:0.287 lr:2.9e-05 updt_s:0.046 data_s:0.238
+INFO 2025-08-06 15:12:24 ts/train.py:232 step:65K smpl:517K ep:1K epch:25.84 loss:0.013 grdn:0.288 lr:2.8e-05 updt_s:0.051 data_s:0.232
+INFO 2025-08-06 15:13:20 ts/train.py:232 step:65K smpl:518K ep:1K epch:25.92 loss:0.014 grdn:0.313 lr:2.8e-05 updt_s:0.048 data_s:0.233
+INFO 2025-08-06 15:14:17 ts/train.py:232 step:65K smpl:520K ep:1K epch:26.00 loss:0.014 grdn:0.311 lr:2.8e-05 updt_s:0.055 data_s:0.228
+INFO 2025-08-06 15:15:14 ts/train.py:232 step:65K smpl:522K ep:1K epch:26.08 loss:0.014 grdn:0.314 lr:2.7e-05 updt_s:0.056 data_s:0.227
+INFO 2025-08-06 15:16:11 ts/train.py:232 step:65K smpl:523K ep:1K epch:26.16 loss:0.014 grdn:0.318 lr:2.7e-05 updt_s:0.057 data_s:0.226
+INFO 2025-08-06 15:17:07 ts/train.py:232 step:66K smpl:525K ep:1K epch:26.24 loss:0.014 grdn:0.310 lr:2.7e-05 updt_s:0.057 data_s:0.226
+INFO 2025-08-06 15:18:04 ts/train.py:232 step:66K smpl:526K ep:1K epch:26.32 loss:0.014 grdn:0.308 lr:2.7e-05 updt_s:0.062 data_s:0.221
+INFO 2025-08-06 15:19:01 ts/train.py:232 step:66K smpl:528K ep:1K epch:26.40 loss:0.014 grdn:0.310 lr:2.6e-05 updt_s:0.060 data_s:0.223
+INFO 2025-08-06 15:19:57 ts/train.py:232 step:66K smpl:530K ep:1K epch:26.48 loss:0.014 grdn:0.308 lr:2.6e-05 updt_s:0.060 data_s:0.222
+INFO 2025-08-06 15:20:55 ts/train.py:232 step:66K smpl:531K ep:1K epch:26.56 loss:0.014 grdn:0.314 lr:2.6e-05 updt_s:0.059 data_s:0.231
+INFO 2025-08-06 15:21:53 ts/train.py:232 step:67K smpl:533K ep:1K epch:26.64 loss:0.013 grdn:0.307 lr:2.5e-05 updt_s:0.072 data_s:0.214
+INFO 2025-08-06 15:22:49 ts/train.py:232 step:67K smpl:534K ep:1K epch:26.72 loss:0.013 grdn:0.310 lr:2.5e-05 updt_s:0.073 data_s:0.210
+INFO 2025-08-06 15:23:46 ts/train.py:232 step:67K smpl:536K ep:1K epch:26.80 loss:0.012 grdn:0.293 lr:2.5e-05 updt_s:0.071 data_s:0.213
+INFO 2025-08-06 15:24:43 ts/train.py:232 step:67K smpl:538K ep:1K epch:26.88 loss:0.013 grdn:0.285 lr:2.5e-05 updt_s:0.072 data_s:0.212
+INFO 2025-08-06 15:25:40 ts/train.py:232 step:67K smpl:539K ep:1K epch:26.96 loss:0.015 grdn:0.330 lr:2.4e-05 updt_s:0.073 data_s:0.211
+INFO 2025-08-06 15:26:37 ts/train.py:232 step:68K smpl:541K ep:1K epch:27.04 loss:0.013 grdn:0.302 lr:2.4e-05 updt_s:0.071 data_s:0.211
+INFO 2025-08-06 15:27:33 ts/train.py:232 step:68K smpl:542K ep:1K epch:27.12 loss:0.014 grdn:0.307 lr:2.4e-05 updt_s:0.073 data_s:0.209
+INFO 2025-08-06 15:28:30 ts/train.py:232 step:68K smpl:544K ep:1K epch:27.20 loss:0.014 grdn:0.310 lr:2.4e-05 updt_s:0.072 data_s:0.211
+INFO 2025-08-06 15:29:27 ts/train.py:232 step:68K smpl:546K ep:1K epch:27.28 loss:0.013 grdn:0.319 lr:2.3e-05 updt_s:0.072 data_s:0.211
+INFO 2025-08-06 15:30:23 ts/train.py:232 step:68K smpl:547K ep:1K epch:27.36 loss:0.013 grdn:0.301 lr:2.3e-05 updt_s:0.073 data_s:0.211
+INFO 2025-08-06 15:31:20 ts/train.py:232 step:69K smpl:549K ep:1K epch:27.44 loss:0.013 grdn:0.310 lr:2.3e-05 updt_s:0.072 data_s:0.212
+INFO 2025-08-06 15:32:17 ts/train.py:232 step:69K smpl:550K ep:1K epch:27.52 loss:0.013 grdn:0.312 lr:2.2e-05 updt_s:0.071 data_s:0.210
+INFO 2025-08-06 15:33:14 ts/train.py:232 step:69K smpl:552K ep:1K epch:27.60 loss:0.014 grdn:0.332 lr:2.2e-05 updt_s:0.057 data_s:0.230
+INFO 2025-08-06 15:34:12 ts/train.py:232 step:69K smpl:554K ep:1K epch:27.68 loss:0.014 grdn:0.309 lr:2.2e-05 updt_s:0.060 data_s:0.226
+INFO 2025-08-06 15:35:09 ts/train.py:232 step:69K smpl:555K ep:1K epch:27.76 loss:0.013 grdn:0.296 lr:2.2e-05 updt_s:0.063 data_s:0.223
+INFO 2025-08-06 15:36:06 ts/train.py:232 step:70K smpl:557K ep:1K epch:27.84 loss:0.013 grdn:0.299 lr:2.1e-05 updt_s:0.067 data_s:0.217
+INFO 2025-08-06 15:37:03 ts/train.py:232 step:70K smpl:558K ep:1K epch:27.92 loss:0.013 grdn:0.299 lr:2.1e-05 updt_s:0.071 data_s:0.214
+INFO 2025-08-06 15:38:00 ts/train.py:232 step:70K smpl:560K ep:1K epch:28.00 loss:0.013 grdn:0.293 lr:2.1e-05 updt_s:0.073 data_s:0.210
+INFO 2025-08-06 15:38:57 ts/train.py:232 step:70K smpl:562K ep:1K epch:28.08 loss:0.014 grdn:0.306 lr:2.1e-05 updt_s:0.073 data_s:0.210
+INFO 2025-08-06 15:39:54 ts/train.py:232 step:70K smpl:563K ep:1K epch:28.16 loss:0.013 grdn:0.315 lr:2.0e-05 updt_s:0.073 data_s:0.211
+INFO 2025-08-06 15:40:50 ts/train.py:232 step:71K smpl:565K ep:1K epch:28.24 loss:0.012 grdn:0.297 lr:2.0e-05 updt_s:0.073 data_s:0.210
+INFO 2025-08-06 15:41:47 ts/train.py:232 step:71K smpl:566K ep:1K epch:28.32 loss:0.012 grdn:0.288 lr:2.0e-05 updt_s:0.072 data_s:0.210
+INFO 2025-08-06 15:42:44 ts/train.py:232 step:71K smpl:568K ep:1K epch:28.40 loss:0.013 grdn:0.291 lr:2.0e-05 updt_s:0.070 data_s:0.212
+INFO 2025-08-06 15:43:40 ts/train.py:232 step:71K smpl:570K ep:1K epch:28.48 loss:0.013 grdn:0.301 lr:1.9e-05 updt_s:0.072 data_s:0.210
+INFO 2025-08-06 15:44:37 ts/train.py:232 step:71K smpl:571K ep:1K epch:28.56 loss:0.013 grdn:0.301 lr:1.9e-05 updt_s:0.064 data_s:0.220
+INFO 2025-08-06 15:45:34 ts/train.py:232 step:72K smpl:573K ep:1K epch:28.64 loss:0.013 grdn:0.298 lr:1.9e-05 updt_s:0.058 data_s:0.224
+INFO 2025-08-06 15:46:30 ts/train.py:232 step:72K smpl:574K ep:1K epch:28.72 loss:0.014 grdn:0.316 lr:1.9e-05 updt_s:0.069 data_s:0.214
+INFO 2025-08-06 15:47:27 ts/train.py:232 step:72K smpl:576K ep:1K epch:28.80 loss:0.013 grdn:0.307 lr:1.8e-05 updt_s:0.073 data_s:0.209
+INFO 2025-08-06 15:48:23 ts/train.py:232 step:72K smpl:578K ep:1K epch:28.88 loss:0.013 grdn:0.292 lr:1.8e-05 updt_s:0.074 data_s:0.207
+INFO 2025-08-06 15:49:19 ts/train.py:232 step:72K smpl:579K ep:1K epch:28.96 loss:0.013 grdn:0.300 lr:1.8e-05 updt_s:0.076 data_s:0.204
+INFO 2025-08-06 15:50:15 ts/train.py:232 step:73K smpl:581K ep:1K epch:29.04 loss:0.013 grdn:0.301 lr:1.8e-05 updt_s:0.075 data_s:0.205
+INFO 2025-08-06 15:51:11 ts/train.py:232 step:73K smpl:582K ep:1K epch:29.12 loss:0.013 grdn:0.303 lr:1.7e-05 updt_s:0.073 data_s:0.207
+INFO 2025-08-06 15:52:07 ts/train.py:232 step:73K smpl:584K ep:1K epch:29.20 loss:0.012 grdn:0.284 lr:1.7e-05 updt_s:0.073 data_s:0.205
+INFO 2025-08-06 15:53:03 ts/train.py:232 step:73K smpl:586K ep:1K epch:29.28 loss:0.012 grdn:0.283 lr:1.7e-05 updt_s:0.073 data_s:0.209
+INFO 2025-08-06 15:54:00 ts/train.py:232 step:73K smpl:587K ep:1K epch:29.36 loss:0.012 grdn:0.286 lr:1.7e-05 updt_s:0.073 data_s:0.209
+INFO 2025-08-06 15:54:56 ts/train.py:232 step:74K smpl:589K ep:1K epch:29.44 loss:0.013 grdn:0.300 lr:1.7e-05 updt_s:0.074 data_s:0.204
+INFO 2025-08-06 15:55:52 ts/train.py:232 step:74K smpl:590K ep:1K epch:29.52 loss:0.012 grdn:0.301 lr:1.6e-05 updt_s:0.060 data_s:0.222
+INFO 2025-08-06 15:56:49 ts/train.py:232 step:74K smpl:592K ep:1K epch:29.60 loss:0.013 grdn:0.298 lr:1.6e-05 updt_s:0.049 data_s:0.235
+INFO 2025-08-06 15:57:46 ts/train.py:232 step:74K smpl:594K ep:1K epch:29.68 loss:0.013 grdn:0.284 lr:1.6e-05 updt_s:0.060 data_s:0.223
+INFO 2025-08-06 15:58:43 ts/train.py:232 step:74K smpl:595K ep:1K epch:29.76 loss:0.012 grdn:0.287 lr:1.6e-05 updt_s:0.053 data_s:0.231
+INFO 2025-08-06 15:59:39 ts/train.py:232 step:75K smpl:597K ep:1K epch:29.84 loss:0.013 grdn:0.305 lr:1.5e-05 updt_s:0.057 data_s:0.224
+INFO 2025-08-06 16:00:36 ts/train.py:232 step:75K smpl:598K ep:1K epch:29.92 loss:0.012 grdn:0.301 lr:1.5e-05 updt_s:0.058 data_s:0.226
+INFO 2025-08-06 16:01:32 ts/train.py:232 step:75K smpl:600K ep:2K epch:30.00 loss:0.012 grdn:0.294 lr:1.5e-05 updt_s:0.051 data_s:0.230
+INFO 2025-08-06 16:02:29 ts/train.py:232 step:75K smpl:602K ep:2K epch:30.08 loss:0.013 grdn:0.302 lr:1.5e-05 updt_s:0.067 data_s:0.218
+INFO 2025-08-06 16:03:26 ts/train.py:232 step:75K smpl:603K ep:2K epch:30.16 loss:0.013 grdn:0.324 lr:1.4e-05 updt_s:0.072 data_s:0.210
+INFO 2025-08-06 16:04:22 ts/train.py:232 step:76K smpl:605K ep:2K epch:30.24 loss:0.013 grdn:0.298 lr:1.4e-05 updt_s:0.074 data_s:0.205
+INFO 2025-08-06 16:05:18 ts/train.py:232 step:76K smpl:606K ep:2K epch:30.32 loss:0.011 grdn:0.283 lr:1.4e-05 updt_s:0.072 data_s:0.209
+INFO 2025-08-06 16:06:15 ts/train.py:232 step:76K smpl:608K ep:2K epch:30.40 loss:0.012 grdn:0.304 lr:1.4e-05 updt_s:0.073 data_s:0.209
+INFO 2025-08-06 16:07:12 ts/train.py:232 step:76K smpl:610K ep:2K epch:30.48 loss:0.012 grdn:0.296 lr:1.4e-05 updt_s:0.068 data_s:0.220
+INFO 2025-08-06 16:08:10 ts/train.py:232 step:76K smpl:611K ep:2K epch:30.56 loss:0.013 grdn:0.301 lr:1.3e-05 updt_s:0.051 data_s:0.239
+INFO 2025-08-06 16:09:09 ts/train.py:232 step:77K smpl:613K ep:2K epch:30.64 loss:0.013 grdn:0.296 lr:1.3e-05 updt_s:0.047 data_s:0.243
+INFO 2025-08-06 16:10:07 ts/train.py:232 step:77K smpl:614K ep:2K epch:30.72 loss:0.013 grdn:0.297 lr:1.3e-05 updt_s:0.052 data_s:0.240
+INFO 2025-08-06 16:11:05 ts/train.py:232 step:77K smpl:616K ep:2K epch:30.80 loss:0.011 grdn:0.279 lr:1.3e-05 updt_s:0.060 data_s:0.231
+INFO 2025-08-06 16:12:04 ts/train.py:232 step:77K smpl:618K ep:2K epch:30.88 loss:0.012 grdn:0.286 lr:1.3e-05 updt_s:0.071 data_s:0.220
+INFO 2025-08-06 16:13:02 ts/train.py:232 step:77K smpl:619K ep:2K epch:30.96 loss:0.012 grdn:0.284 lr:1.2e-05 updt_s:0.073 data_s:0.217
+INFO 2025-08-06 16:14:00 ts/train.py:232 step:78K smpl:621K ep:2K epch:31.04 loss:0.013 grdn:0.293 lr:1.2e-05 updt_s:0.074 data_s:0.216
+INFO 2025-08-06 16:14:58 ts/train.py:232 step:78K smpl:622K ep:2K epch:31.12 loss:0.012 grdn:0.295 lr:1.2e-05 updt_s:0.074 data_s:0.216
+INFO 2025-08-06 16:15:56 ts/train.py:232 step:78K smpl:624K ep:2K epch:31.20 loss:0.012 grdn:0.292 lr:1.2e-05 updt_s:0.072 data_s:0.217
+INFO 2025-08-06 16:16:53 ts/train.py:232 step:78K smpl:626K ep:2K epch:31.28 loss:0.014 grdn:0.303 lr:1.1e-05 updt_s:0.074 data_s:0.215
+INFO 2025-08-06 16:17:51 ts/train.py:232 step:78K smpl:627K ep:2K epch:31.36 loss:0.012 grdn:0.281 lr:1.1e-05 updt_s:0.073 data_s:0.216
+INFO 2025-08-06 16:18:50 ts/train.py:232 step:79K smpl:629K ep:2K epch:31.44 loss:0.012 grdn:0.291 lr:1.1e-05 updt_s:0.070 data_s:0.220
+INFO 2025-08-06 16:19:48 ts/train.py:232 step:79K smpl:630K ep:2K epch:31.52 loss:0.011 grdn:0.273 lr:1.1e-05 updt_s:0.063 data_s:0.228
+INFO 2025-08-06 16:20:44 ts/train.py:232 step:79K smpl:632K ep:2K epch:31.60 loss:0.012 grdn:0.296 lr:1.1e-05 updt_s:0.069 data_s:0.210
+INFO 2025-08-06 16:21:39 ts/train.py:232 step:79K smpl:634K ep:2K epch:31.68 loss:0.012 grdn:0.307 lr:1.0e-05 updt_s:0.071 data_s:0.206
+INFO 2025-08-06 16:22:35 ts/train.py:232 step:79K smpl:635K ep:2K epch:31.76 loss:0.012 grdn:0.298 lr:1.0e-05 updt_s:0.071 data_s:0.206
+INFO 2025-08-06 16:23:30 ts/train.py:232 step:80K smpl:637K ep:2K epch:31.84 loss:0.012 grdn:0.290 lr:1.0e-05 updt_s:0.071 data_s:0.207
+INFO 2025-08-06 16:24:26 ts/train.py:232 step:80K smpl:638K ep:2K epch:31.92 loss:0.014 grdn:0.305 lr:9.9e-06 updt_s:0.072 data_s:0.204
+INFO 2025-08-06 16:25:22 ts/train.py:232 step:80K smpl:640K ep:2K epch:32.00 loss:0.013 grdn:0.314 lr:9.7e-06 updt_s:0.066 data_s:0.213
+INFO 2025-08-06 16:25:22 ts/train.py:241 Checkpoint policy after step 80000
+INFO 2025-08-06 16:26:24 ts/train.py:232 step:80K smpl:642K ep:2K epch:32.08 loss:0.012 grdn:0.288 lr:9.5e-06 updt_s:0.066 data_s:0.218
+INFO 2025-08-06 16:27:20 ts/train.py:232 step:80K smpl:643K ep:2K epch:32.16 loss:0.013 grdn:0.299 lr:9.4e-06 updt_s:0.064 data_s:0.216
+INFO 2025-08-06 16:28:17 ts/train.py:232 step:81K smpl:645K ep:2K epch:32.24 loss:0.012 grdn:0.281 lr:9.2e-06 updt_s:0.068 data_s:0.213
+INFO 2025-08-06 16:29:13 ts/train.py:232 step:81K smpl:646K ep:2K epch:32.32 loss:0.012 grdn:0.280 lr:9.0e-06 updt_s:0.073 data_s:0.209
+INFO 2025-08-06 16:30:09 ts/train.py:232 step:81K smpl:648K ep:2K epch:32.40 loss:0.013 grdn:0.300 lr:8.8e-06 updt_s:0.072 data_s:0.207
+INFO 2025-08-06 16:31:05 ts/train.py:232 step:81K smpl:650K ep:2K epch:32.48 loss:0.013 grdn:0.289 lr:8.6e-06 updt_s:0.068 data_s:0.212
+INFO 2025-08-06 16:32:01 ts/train.py:232 step:81K smpl:651K ep:2K epch:32.56 loss:0.012 grdn:0.271 lr:8.5e-06 updt_s:0.061 data_s:0.217
+INFO 2025-08-06 16:32:57 ts/train.py:232 step:82K smpl:653K ep:2K epch:32.64 loss:0.013 grdn:0.321 lr:8.3e-06 updt_s:0.071 data_s:0.209
+INFO 2025-08-06 16:33:52 ts/train.py:232 step:82K smpl:654K ep:2K epch:32.72 loss:0.012 grdn:0.293 lr:8.1e-06 updt_s:0.072 data_s:0.203
+INFO 2025-08-06 16:34:47 ts/train.py:232 step:82K smpl:656K ep:2K epch:32.80 loss:0.011 grdn:0.254 lr:7.9e-06 updt_s:0.073 data_s:0.202
+INFO 2025-08-06 16:35:43 ts/train.py:232 step:82K smpl:658K ep:2K epch:32.88 loss:0.012 grdn:0.291 lr:7.8e-06 updt_s:0.073 data_s:0.204
+INFO 2025-08-06 16:36:38 ts/train.py:232 step:82K smpl:659K ep:2K epch:32.96 loss:0.012 grdn:0.298 lr:7.6e-06 updt_s:0.073 data_s:0.203
+INFO 2025-08-06 16:37:34 ts/train.py:232 step:83K smpl:661K ep:2K epch:33.04 loss:0.012 grdn:0.288 lr:7.4e-06 updt_s:0.073 data_s:0.205
+INFO 2025-08-06 16:38:29 ts/train.py:232 step:83K smpl:662K ep:2K epch:33.12 loss:0.012 grdn:0.291 lr:7.3e-06 updt_s:0.073 data_s:0.203
+INFO 2025-08-06 16:39:24 ts/train.py:232 step:83K smpl:664K ep:2K epch:33.20 loss:0.012 grdn:0.285 lr:7.1e-06 updt_s:0.072 data_s:0.203
+INFO 2025-08-06 16:40:19 ts/train.py:232 step:83K smpl:666K ep:2K epch:33.28 loss:0.013 grdn:0.299 lr:7.0e-06 updt_s:0.072 data_s:0.204
+INFO 2025-08-06 16:41:15 ts/train.py:232 step:83K smpl:667K ep:2K epch:33.36 loss:0.012 grdn:0.277 lr:6.8e-06 updt_s:0.073 data_s:0.204
+INFO 2025-08-06 16:42:11 ts/train.py:232 step:84K smpl:669K ep:2K epch:33.44 loss:0.012 grdn:0.282 lr:6.6e-06 updt_s:0.067 data_s:0.214
+INFO 2025-08-06 16:43:08 ts/train.py:232 step:84K smpl:670K ep:2K epch:33.52 loss:0.011 grdn:0.271 lr:6.5e-06 updt_s:0.062 data_s:0.222
+INFO 2025-08-06 16:44:05 ts/train.py:232 step:84K smpl:672K ep:2K epch:33.60 loss:0.013 grdn:0.294 lr:6.3e-06 updt_s:0.059 data_s:0.226
+INFO 2025-08-06 16:45:02 ts/train.py:232 step:84K smpl:674K ep:2K epch:33.68 loss:0.012 grdn:0.284 lr:6.2e-06 updt_s:0.057 data_s:0.228
+INFO 2025-08-06 16:45:59 ts/train.py:232 step:84K smpl:675K ep:2K epch:33.76 loss:0.011 grdn:0.277 lr:6.0e-06 updt_s:0.063 data_s:0.220
+INFO 2025-08-06 16:46:56 ts/train.py:232 step:85K smpl:677K ep:2K epch:33.84 loss:0.011 grdn:0.269 lr:5.9e-06 updt_s:0.064 data_s:0.220
+INFO 2025-08-06 16:47:53 ts/train.py:232 step:85K smpl:678K ep:2K epch:33.92 loss:0.012 grdn:0.265 lr:5.7e-06 updt_s:0.066 data_s:0.218
+INFO 2025-08-06 16:48:50 ts/train.py:232 step:85K smpl:680K ep:2K epch:34.00 loss:0.012 grdn:0.282 lr:5.6e-06 updt_s:0.062 data_s:0.221
+INFO 2025-08-06 16:49:46 ts/train.py:232 step:85K smpl:682K ep:2K epch:34.08 loss:0.012 grdn:0.292 lr:5.4e-06 updt_s:0.062 data_s:0.222
+INFO 2025-08-06 16:50:43 ts/train.py:232 step:85K smpl:683K ep:2K epch:34.16 loss:0.013 grdn:0.290 lr:5.3e-06 updt_s:0.069 data_s:0.214
+INFO 2025-08-06 16:51:40 ts/train.py:232 step:86K smpl:685K ep:2K epch:34.24 loss:0.012 grdn:0.285 lr:5.1e-06 updt_s:0.073 data_s:0.210
+INFO 2025-08-06 16:52:36 ts/train.py:232 step:86K smpl:686K ep:2K epch:34.32 loss:0.012 grdn:0.299 lr:5.0e-06 updt_s:0.075 data_s:0.208
+INFO 2025-08-06 16:53:34 ts/train.py:232 step:86K smpl:688K ep:2K epch:34.40 loss:0.012 grdn:0.275 lr:4.9e-06 updt_s:0.071 data_s:0.214
+INFO 2025-08-06 16:54:31 ts/train.py:232 step:86K smpl:690K ep:2K epch:34.48 loss:0.012 grdn:0.290 lr:4.7e-06 updt_s:0.052 data_s:0.236
+INFO 2025-08-06 16:55:29 ts/train.py:232 step:86K smpl:691K ep:2K epch:34.56 loss:0.013 grdn:0.293 lr:4.6e-06 updt_s:0.061 data_s:0.227
+INFO 2025-08-06 16:56:27 ts/train.py:232 step:87K smpl:693K ep:2K epch:34.64 loss:0.011 grdn:0.277 lr:4.5e-06 updt_s:0.066 data_s:0.221
+INFO 2025-08-06 16:57:24 ts/train.py:232 step:87K smpl:694K ep:2K epch:34.72 loss:0.011 grdn:0.287 lr:4.3e-06 updt_s:0.075 data_s:0.212
+INFO 2025-08-06 16:58:22 ts/train.py:232 step:87K smpl:696K ep:2K epch:34.80 loss:0.012 grdn:0.295 lr:4.2e-06 updt_s:0.074 data_s:0.212
+INFO 2025-08-06 16:59:19 ts/train.py:232 step:87K smpl:698K ep:2K epch:34.88 loss:0.011 grdn:0.277 lr:4.1e-06 updt_s:0.075 data_s:0.212
+INFO 2025-08-06 17:00:17 ts/train.py:232 step:87K smpl:699K ep:2K epch:34.96 loss:0.012 grdn:0.282 lr:4.0e-06 updt_s:0.074 data_s:0.214
+INFO 2025-08-06 17:01:14 ts/train.py:232 step:88K smpl:701K ep:2K epch:35.04 loss:0.011 grdn:0.285 lr:3.8e-06 updt_s:0.075 data_s:0.214
+INFO 2025-08-06 17:02:12 ts/train.py:232 step:88K smpl:702K ep:2K epch:35.12 loss:0.013 grdn:0.301 lr:3.7e-06 updt_s:0.074 data_s:0.213
+INFO 2025-08-06 17:03:09 ts/train.py:232 step:88K smpl:704K ep:2K epch:35.20 loss:0.011 grdn:0.265 lr:3.6e-06 updt_s:0.074 data_s:0.211
+INFO 2025-08-06 17:04:07 ts/train.py:232 step:88K smpl:706K ep:2K epch:35.28 loss:0.012 grdn:0.285 lr:3.5e-06 updt_s:0.075 data_s:0.213
+INFO 2025-08-06 17:05:04 ts/train.py:232 step:88K smpl:707K ep:2K epch:35.36 loss:0.012 grdn:0.289 lr:3.4e-06 updt_s:0.074 data_s:0.212
+INFO 2025-08-06 17:06:02 ts/train.py:232 step:89K smpl:709K ep:2K epch:35.44 loss:0.012 grdn:0.283 lr:3.3e-06 updt_s:0.064 data_s:0.226
+INFO 2025-08-06 17:06:59 ts/train.py:232 step:89K smpl:710K ep:2K epch:35.52 loss:0.012 grdn:0.275 lr:3.1e-06 updt_s:0.063 data_s:0.223
+INFO 2025-08-06 17:07:57 ts/train.py:232 step:89K smpl:712K ep:2K epch:35.60 loss:0.012 grdn:0.287 lr:3.0e-06 updt_s:0.069 data_s:0.221
+INFO 2025-08-06 17:08:55 ts/train.py:232 step:89K smpl:714K ep:2K epch:35.68 loss:0.011 grdn:0.286 lr:2.9e-06 updt_s:0.069 data_s:0.217
+INFO 2025-08-06 17:09:52 ts/train.py:232 step:89K smpl:715K ep:2K epch:35.76 loss:0.011 grdn:0.285 lr:2.8e-06 updt_s:0.070 data_s:0.216
+INFO 2025-08-06 17:10:49 ts/train.py:232 step:90K smpl:717K ep:2K epch:35.84 loss:0.013 grdn:0.316 lr:2.7e-06 updt_s:0.071 data_s:0.215
+INFO 2025-08-06 17:11:47 ts/train.py:232 step:90K smpl:718K ep:2K epch:35.92 loss:0.012 grdn:0.286 lr:2.6e-06 updt_s:0.072 data_s:0.216
+INFO 2025-08-06 17:12:45 ts/train.py:232 step:90K smpl:720K ep:2K epch:36.00 loss:0.011 grdn:0.291 lr:2.5e-06 updt_s:0.072 data_s:0.216
+INFO 2025-08-06 17:13:42 ts/train.py:232 step:90K smpl:722K ep:2K epch:36.08 loss:0.013 grdn:0.307 lr:2.4e-06 updt_s:0.072 data_s:0.215
+INFO 2025-08-06 17:14:39 ts/train.py:232 step:90K smpl:723K ep:2K epch:36.16 loss:0.013 grdn:0.290 lr:2.3e-06 updt_s:0.071 data_s:0.216
+INFO 2025-08-06 17:15:37 ts/train.py:232 step:91K smpl:725K ep:2K epch:36.24 loss:0.011 grdn:0.294 lr:2.2e-06 updt_s:0.070 data_s:0.216
+INFO 2025-08-06 17:16:34 ts/train.py:232 step:91K smpl:726K ep:2K epch:36.32 loss:0.011 grdn:0.283 lr:2.1e-06 updt_s:0.071 data_s:0.216
+INFO 2025-08-06 17:17:33 ts/train.py:232 step:91K smpl:728K ep:2K epch:36.40 loss:0.012 grdn:0.288 lr:2.0e-06 updt_s:0.059 data_s:0.235
+INFO 2025-08-06 17:18:32 ts/train.py:232 step:91K smpl:730K ep:2K epch:36.48 loss:0.011 grdn:0.270 lr:2.0e-06 updt_s:0.059 data_s:0.235
+INFO 2025-08-06 17:19:31 ts/train.py:232 step:91K smpl:731K ep:2K epch:36.56 loss:0.012 grdn:0.277 lr:1.9e-06 updt_s:0.070 data_s:0.224
+INFO 2025-08-06 17:20:29 ts/train.py:232 step:92K smpl:733K ep:2K epch:36.64 loss:0.012 grdn:0.288 lr:1.8e-06 updt_s:0.073 data_s:0.216
+INFO 2025-08-06 17:21:27 ts/train.py:232 step:92K smpl:734K ep:2K epch:36.72 loss:0.012 grdn:0.297 lr:1.7e-06 updt_s:0.071 data_s:0.218
+INFO 2025-08-06 17:22:25 ts/train.py:232 step:92K smpl:736K ep:2K epch:36.80 loss:0.012 grdn:0.285 lr:1.6e-06 updt_s:0.072 data_s:0.216
+INFO 2025-08-06 17:23:23 ts/train.py:232 step:92K smpl:738K ep:2K epch:36.88 loss:0.011 grdn:0.278 lr:1.5e-06 updt_s:0.073 data_s:0.216
+INFO 2025-08-06 17:24:20 ts/train.py:232 step:92K smpl:739K ep:2K epch:36.96 loss:0.011 grdn:0.257 lr:1.5e-06 updt_s:0.071 data_s:0.218
+INFO 2025-08-06 17:25:18 ts/train.py:232 step:93K smpl:741K ep:2K epch:37.04 loss:0.013 grdn:0.289 lr:1.4e-06 updt_s:0.072 data_s:0.217
+INFO 2025-08-06 17:26:16 ts/train.py:232 step:93K smpl:742K ep:2K epch:37.12 loss:0.012 grdn:0.284 lr:1.3e-06 updt_s:0.073 data_s:0.214
+INFO 2025-08-06 17:27:13 ts/train.py:232 step:93K smpl:744K ep:2K epch:37.20 loss:0.012 grdn:0.281 lr:1.3e-06 updt_s:0.074 data_s:0.214
+INFO 2025-08-06 17:28:11 ts/train.py:232 step:93K smpl:746K ep:2K epch:37.28 loss:0.011 grdn:0.267 lr:1.2e-06 updt_s:0.073 data_s:0.214
+INFO 2025-08-06 17:29:09 ts/train.py:232 step:93K smpl:747K ep:2K epch:37.36 loss:0.010 grdn:0.267 lr:1.1e-06 updt_s:0.069 data_s:0.222
+INFO 2025-08-06 17:30:08 ts/train.py:232 step:94K smpl:749K ep:2K epch:37.44 loss:0.011 grdn:0.272 lr:1.0e-06 updt_s:0.056 data_s:0.235
+INFO 2025-08-06 17:31:06 ts/train.py:232 step:94K smpl:750K ep:2K epch:37.52 loss:0.012 grdn:0.273 lr:9.9e-07 updt_s:0.070 data_s:0.224
+INFO 2025-08-06 17:32:05 ts/train.py:232 step:94K smpl:752K ep:2K epch:37.60 loss:0.011 grdn:0.289 lr:9.2e-07 updt_s:0.072 data_s:0.219
+INFO 2025-08-06 17:33:03 ts/train.py:232 step:94K smpl:754K ep:2K epch:37.68 loss:0.012 grdn:0.278 lr:8.6e-07 updt_s:0.072 data_s:0.219
+INFO 2025-08-06 17:34:01 ts/train.py:232 step:94K smpl:755K ep:2K epch:37.76 loss:0.011 grdn:0.279 lr:8.1e-07 updt_s:0.073 data_s:0.217
+INFO 2025-08-06 17:35:00 ts/train.py:232 step:95K smpl:757K ep:2K epch:37.84 loss:0.013 grdn:0.293 lr:7.5e-07 updt_s:0.074 data_s:0.217
+INFO 2025-08-06 17:35:58 ts/train.py:232 step:95K smpl:758K ep:2K epch:37.92 loss:0.011 grdn:0.270 lr:7.0e-07 updt_s:0.074 data_s:0.218
+INFO 2025-08-06 17:36:56 ts/train.py:232 step:95K smpl:760K ep:2K epch:38.00 loss:0.012 grdn:0.272 lr:6.5e-07 updt_s:0.074 data_s:0.217
+INFO 2025-08-06 17:37:54 ts/train.py:232 step:95K smpl:762K ep:2K epch:38.08 loss:0.012 grdn:0.283 lr:6.0e-07 updt_s:0.073 data_s:0.217
+INFO 2025-08-06 17:38:53 ts/train.py:232 step:95K smpl:763K ep:2K epch:38.16 loss:0.011 grdn:0.268 lr:5.5e-07 updt_s:0.073 data_s:0.217
+INFO 2025-08-06 17:39:51 ts/train.py:232 step:96K smpl:765K ep:2K epch:38.24 loss:0.012 grdn:0.277 lr:5.0e-07 updt_s:0.074 data_s:0.219
+INFO 2025-08-06 17:40:49 ts/train.py:232 step:96K smpl:766K ep:2K epch:38.32 loss:0.012 grdn:0.284 lr:4.6e-07 updt_s:0.072 data_s:0.218
+INFO 2025-08-06 17:41:50 ts/train.py:232 step:96K smpl:768K ep:2K epch:38.40 loss:0.011 grdn:0.278 lr:4.2e-07 updt_s:0.062 data_s:0.238
+INFO 2025-08-06 17:42:48 ts/train.py:232 step:96K smpl:770K ep:2K epch:38.48 loss:0.012 grdn:0.293 lr:3.8e-07 updt_s:0.063 data_s:0.228
+INFO 2025-08-06 17:43:46 ts/train.py:232 step:96K smpl:771K ep:2K epch:38.56 loss:0.011 grdn:0.274 lr:3.4e-07 updt_s:0.061 data_s:0.232
+INFO 2025-08-06 17:44:44 ts/train.py:232 step:97K smpl:773K ep:2K epch:38.64 loss:0.010 grdn:0.260 lr:3.0e-07 updt_s:0.061 data_s:0.229
+INFO 2025-08-06 17:45:43 ts/train.py:232 step:97K smpl:774K ep:2K epch:38.72 loss:0.012 grdn:0.279 lr:2.7e-07 updt_s:0.062 data_s:0.229
+INFO 2025-08-06 17:46:41 ts/train.py:232 step:97K smpl:776K ep:2K epch:38.80 loss:0.011 grdn:0.274 lr:2.4e-07 updt_s:0.059 data_s:0.232
+INFO 2025-08-06 17:47:39 ts/train.py:232 step:97K smpl:778K ep:2K epch:38.88 loss:0.011 grdn:0.262 lr:2.1e-07 updt_s:0.046 data_s:0.244
+INFO 2025-08-06 17:48:38 ts/train.py:232 step:97K smpl:779K ep:2K epch:38.96 loss:0.012 grdn:0.287 lr:1.8e-07 updt_s:0.050 data_s:0.245
+INFO 2025-08-06 17:49:37 ts/train.py:232 step:98K smpl:781K ep:2K epch:39.04 loss:0.012 grdn:0.274 lr:1.6e-07 updt_s:0.052 data_s:0.240
+INFO 2025-08-06 17:50:36 ts/train.py:232 step:98K smpl:782K ep:2K epch:39.12 loss:0.012 grdn:0.282 lr:1.3e-07 updt_s:0.059 data_s:0.233
+INFO 2025-08-06 17:51:34 ts/train.py:232 step:98K smpl:784K ep:2K epch:39.20 loss:0.012 grdn:0.276 lr:1.1e-07 updt_s:0.064 data_s:0.228
+INFO 2025-08-06 17:52:32 ts/train.py:232 step:98K smpl:786K ep:2K epch:39.28 loss:0.012 grdn:0.282 lr:9.0e-08 updt_s:0.063 data_s:0.228
+INFO 2025-08-06 17:53:33 ts/train.py:232 step:98K smpl:787K ep:2K epch:39.36 loss:0.012 grdn:0.284 lr:7.2e-08 updt_s:0.065 data_s:0.235
+INFO 2025-08-06 17:54:32 ts/train.py:232 step:99K smpl:789K ep:2K epch:39.44 loss:0.012 grdn:0.278 lr:5.6e-08 updt_s:0.063 data_s:0.233
+INFO 2025-08-06 17:55:30 ts/train.py:232 step:99K smpl:790K ep:2K epch:39.52 loss:0.012 grdn:0.295 lr:4.2e-08 updt_s:0.062 data_s:0.229
+INFO 2025-08-06 17:56:29 ts/train.py:232 step:99K smpl:792K ep:2K epch:39.60 loss:0.011 grdn:0.277 lr:3.0e-08 updt_s:0.065 data_s:0.227
+INFO 2025-08-06 17:57:27 ts/train.py:232 step:99K smpl:794K ep:2K epch:39.68 loss:0.011 grdn:0.278 lr:2.0e-08 updt_s:0.069 data_s:0.222
+INFO 2025-08-06 17:58:26 ts/train.py:232 step:99K smpl:795K ep:2K epch:39.76 loss:0.011 grdn:0.293 lr:1.2e-08 updt_s:0.059 data_s:0.235
+INFO 2025-08-06 17:59:24 ts/train.py:232 step:100K smpl:797K ep:2K epch:39.84 loss:0.012 grdn:0.288 lr:6.3e-09 updt_s:0.062 data_s:0.229
+INFO 2025-08-06 18:00:23 ts/train.py:232 step:100K smpl:798K ep:2K epch:39.92 loss:0.013 grdn:0.297 lr:2.3e-09 updt_s:0.073 data_s:0.220
+INFO 2025-08-06 18:01:21 ts/train.py:232 step:100K smpl:800K ep:2K epch:40.00 loss:0.011 grdn:0.289 lr:3.3e-10 updt_s:0.072 data_s:0.218
+INFO 2025-08-06 18:01:21 ts/train.py:241 Checkpoint policy after step 100000
+Traceback (most recent call last):
+  File "/projects/p32775/pythonenvs/tac_vla/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/projects/p32775/pythonenvs/tac_vla/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/scripts/train.py", line 295, in <module>
+    main()
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/scripts/train.py", line 291, in main
+    train()
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/configs/parser.py", line 225, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/scripts/train.py", line 243, in train
+    save_checkpoint(checkpoint_dir, step, cfg, policy, optimizer, lr_scheduler)
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/utils/train_utils.py", line 102, in save_checkpoint
+    save_training_state(checkpoint_dir, step, optimizer, scheduler)
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/utils/train_utils.py", line 127, in save_training_state
+    save_optimizer_state(optimizer, save_dir)
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/optim/optimizers.py", line 172, in save_optimizer_state
+    _save_single_optimizer_state(optimizer, save_dir)
+  File "/gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/src/lerobot/optim/optimizers.py", line 180, in _save_single_optimizer_state
+    save_file(flat_state, save_dir / OPTIMIZER_STATE)
+  File "/projects/p32775/pythonenvs/tac_vla/lib/python3.10/site-packages/safetensors/torch.py", line 286, in save_file
+    serialize_file(_flatten(tensors), filename, metadata=metadata)
+safetensors_rust.SafetensorError: Error while serializing: IoError(Os { code: 122, kind: QuotaExceeded, message: "Disk quota exceeded" })

wandb/run-20250806_095353-4m3ikyrf/files/requirements.txt

@@ -0,0 +1,146 @@
+lightning-utilities==0.14.3
+uv==0.7.19
+umap-learn==0.5.9.post2
+torchmetrics==1.7.4
+anndata==0.11.4
+pynndescent==0.5.13
+array-api-compat==1.12.0
+contourpy==1.3.2
+tzdata==2025.2
+filelock==3.18.0
+natsort==8.4.0
+scipy==1.15.3
+kiwisolver==1.4.8
+networkx==3.4.2
+numba==0.61.2
+patsy==1.0.1
+legacy-api-wrap==1.4.1
+pillow==11.3.0
+pytorch-lightning==2.5.2
+cycler==0.12.1
+h5py==3.14.0
+pyparsing==3.2.3
+packaging==25.0
+joblib==1.5.1
+session-info2==0.2
+scikit-learn==1.7.1
+cell-load==0.7.4
+statsmodels==0.14.5
+numpy==2.2.6
+mpmath==1.3.0
+lightning==2.5.2
+fonttools==4.59.0
+typing_extensions==4.14.1
+seaborn==0.13.2
+scanpy==1.11.3
+toml==0.10.2
+matplotlib==3.10.3
+threadpoolctl==3.6.0
+llvmlite==0.44.0
+pandas==2.3.1
+nvidia-cufile-cu12==1.11.1.6
+triton==3.3.1
+charset-normalizer==3.4.2
+sentry-sdk==2.34.1
+nvidia-cuda-runtime-cu12==12.6.77
+pyarrow==21.0.0
+sympy==1.14.0
+pynput==1.8.1
+pfzy==0.3.4
+Jinja2==3.1.6
+pydantic_core==2.33.2
+cmake==4.0.3
+nvidia-cusparse-cu12==12.5.4.2
+gitdb==4.0.12
+aiosignal==1.4.0
+yarl==1.20.1
+platformdirs==4.3.8
+jsonlines==4.0.0
+multiprocess==0.70.16
+certifi==2025.8.3
+inquirerpy==0.3.4
+nvidia-nvjitlink-cu12==12.6.85
+nvidia-cublas-cu12==12.6.4.1
+pydantic==2.11.7
+async-timeout==5.0.1
+annotated-types==0.7.0
+typing-inspect==0.9.0
+imageio==2.37.0
+nvidia-cuda-cupti-cu12==12.6.80
+wcwidth==0.2.13
+six==1.17.0
+tqdm==4.67.1
+zipp==3.23.0
+torchcodec==0.5
+cloudpickle==3.1.1
+lerobot==0.3.2
+Flask==3.1.1
+python-xlib==0.33
+wandb==0.21.0
+urllib3==2.5.0
+nvidia-nccl-cu12==2.26.2
+typing-inspection==0.4.1
+evdev==1.9.2
+prompt_toolkit==3.0.51
+gymnasium==0.29.1
+nvidia-curand-cu12==10.3.7.77
+GitPython==3.1.45
+opencv-python-headless==4.12.0.88
+datasets==3.6.0
+nvidia-cusolver-cu12==11.7.1.2
+termcolor==3.1.0
+mypy_extensions==1.1.0
+idna==3.10
+MarkupSafe==3.0.2
+frozenlist==1.7.0
+regex==2025.7.34
+Werkzeug==3.1.3
+imageio-ffmpeg==0.6.0
+importlib_metadata==8.7.0
+blinker==1.9.0
+psutil==7.0.0
+fsspec==2025.3.0
+deepdiff==8.5.0
+dill==0.3.8
+diffusers==0.34.0
+wheel==0.45.1
+nvidia-nvtx-cu12==12.6.77
+av==15.0.0
+nvidia-cusparselt-cu12==0.6.3
+propcache==0.3.2
+python-can==4.5.0
+msgpack==1.1.1
+einops==0.8.1
+safetensors==0.5.3
+setuptools==78.1.1
+requests==2.32.4
+torchvision==0.22.1
+wrapt==1.17.2
+pyyaml-include==1.4.1
+piper-sdk==0.4.1
+nvidia-cufft-cu12==11.3.0.4
+nvidia-cuda-nvrtc-cu12==12.6.77
+Farama-Notifications==0.0.4
+click==8.2.1
+pyserial==3.5
+smmap==5.0.2
+attrs==25.3.0
+draccus==0.10.0
+mergedeep==1.3.4
+huggingface-hub==0.34.3
+pytz==2025.2
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+itsdangerous==2.2.0
+torch==2.7.1
+multidict==6.6.3
+aiohttp==3.12.15
+protobuf==6.31.1
+hf_transfer==0.1.9
+xxhash==3.5.0
+pip==25.1
+PyYAML==6.0.2
+orderly-set==5.5.0
+nvidia-cudnn-cu12==9.5.1.17
+hf-xet==1.1.5
+lerobot==0.3.2

wandb/run-20250806_095353-4m3ikyrf/files/wandb-metadata.json

@@ -0,0 +1,77 @@
+{
+  "os":  "Linux-4.18.0-553.36.1.el8_10.x86_64-x86_64-with-glibc2.28",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2025-08-06T14:53:53.171298Z",
+  "args":  [
+    "--dataset.repo_id=a1o/pick-blue-cube_sarm",
+    "--policy.type=diffusion",
+    "--output_dir=outputs/train/pick-blue-cube-single-arm_diff",
+    "--job_name=diff_pick-blue-cube-single-arm",
+    "--policy.device=cuda",
+    "--wandb.enable=true",
+    "--policy.repo_id=a1o/diff_pick-blue-cube_sarm_policy"
+  ],
+  "program":  "-m lerobot.scripts.train",
+  "root":  "outputs/train/pick-blue-cube-single-arm_diff",
+  "host":  "qgpu3019",
+  "executable":  "/projects/p32775/pythonenvs/tac_vla/bin/python",
+  "cpu_count":  64,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "540905492480",
+      "used":  "7185981440"
+    }
+  },
+  "memory":  {
+    "total":  "1081810989056"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-8a97223c-63d6-86ef-9b46-7e4225e9933e"
+    }
+  ],
+  "cudaVersion":  "12.8",
+  "slurm":  {
+    "cluster_name":  "quest",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "1",
+    "gpus_on_node":  "1",
+    "gtids":  "0",
+    "job_account":  "p32775",
+    "job_cpus_per_node":  "1",
+    "job_end_time":  "1754527940",
+    "job_gid":  "2000033",
+    "job_gpus":  "1",
+    "job_id":  "971322",
+    "job_name":  "ACT_pick-blue-cube-single-arm",
+    "job_nodelist":  "qgpu3019",
+    "job_num_nodes":  "1",
+    "job_partition":  "gengpu",
+    "job_qos":  "normal",
+    "job_start_time":  "1754491940",
+    "job_uid":  "2000033",
+    "job_user":  "aye8078",
+    "jobid":  "971322",
+    "localid":  "0",
+    "mem_per_node":  "49152",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "qgpu3019",
+    "prio_process":  "0",
+    "procid":  "0",
+    "submit_dir":  "/gpfs/home/aye8078/Documents/Github/tac_vla/slurm",
+    "submit_host":  "quser32",
+    "task_pid":  "4150120",
+    "tasks_per_node":  "1",
+    "topology_addr":  "qgpu3019",
+    "topology_addr_pattern":  "node"
+  },
+  "writerId":  "e37fw9fep90im526hixlw3wga8gghk11"
+}

wandb/run-20250806_095353-4m3ikyrf/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss":0.010989634525612928,"train/steps":100000,"train/samples":800000,"_wandb":{"runtime":29251},"_timestamp":1.7545212814082277e+09,"train/grad_norm":0.2890039833635092,"train/update_s":0.07225841182284057,"train/episodes":2000,"_runtime":29251.296746469,"_step":100000,"train/dataloading_s":0.2181737862713635,"train/epochs":40,"train/lr":3.298127591122574e-10}

wandb/run-20250806_095353-4m3ikyrf/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-08-06T09:53:53.7511107-05:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpqbs8h7d4/port-4150232.txt","pid":4150232,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-08-06T09:53:53.751769328-05:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4150232-4150957-4080849505/socket","Net":"unix"}}
+{"time":"2025-08-06T09:53:53.751893212-05:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":4150232}
+{"time":"2025-08-06T09:53:53.911191845-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-08-06T09:53:53.969313538-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"4m3ikyrf","id":"1(@)"}
+{"time":"2025-08-06T09:53:54.181600316-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"4m3ikyrf","id":"1(@)"}
+{"time":"2025-08-06T18:01:25.553579068-05:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-08-06T18:01:25.622173047-05:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-08-06T18:01:25.622226654-05:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4150232-4150957-4080849505/socket","Net":"unix"}}
+{"time":"2025-08-06T18:01:25.636743904-05:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-08-06T18:01:25.636768139-05:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-08-06T18:01:26.560194675-05:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-08-06T18:01:26.560215115-05:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-08-06T18:01:26.560221187-05:00","level":"INFO","msg":"server is closed"}

wandb/run-20250806_095353-4m3ikyrf/logs/debug-internal.log

@@ -0,0 +1,36 @@
+{"time":"2025-08-06T09:53:53.97048859-05:00","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
+{"time":"2025-08-06T09:53:54.181541434-05:00","level":"INFO","msg":"stream: created new stream","id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.181593892-05:00","level":"INFO","msg":"stream: started","id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.182561483-05:00","level":"INFO","msg":"handler: started","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.182602292-05:00","level":"INFO","msg":"writer: Do: started","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T09:53:54.182749259-05:00","level":"INFO","msg":"sender: started","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T15:22:36.283609749-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:23:21.287384372-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:23:36.285724184-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:23:47.496306298-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:24:23.543803005-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/a10v-1/lerobot/4m3ikyrf/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:25:17.124550591-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/a10v-1/lerobot/4m3ikyrf/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:25:17.794337051-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:26:17.262055087-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:27:19.720003097-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:27:44.671942739-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:27:48.400231348-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/a10v-1/lerobot/4m3ikyrf/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-08-06T15:28:19.10820662-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:28:46.051038776-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:29:57.280536859-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-06T15:30:20.121098493-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:31:42.281292653-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-06T15:32:05.208409505-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:32:29.220382351-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:33:16.719371795-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:33:39.286298851-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:37:44.31879675-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:40:17.910537945-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
+{"time":"2025-08-06T15:48:57.304808267-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-06T16:25:37.13628613-05:00","level":"ERROR","msg":"error adding file to cache","err":"write /home/aye8078/.cache/wandb/artifacts/tmp/2102143582: disk quota exceeded"}
+{"time":"2025-08-06T18:01:25.622284787-05:00","level":"INFO","msg":"stream: closing","id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.32138818-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-06T18:01:26.414301344-05:00","level":"INFO","msg":"handler: closed","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.414318337-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.414434409-05:00","level":"INFO","msg":"sender: closed","stream_id":"4m3ikyrf"}
+{"time":"2025-08-06T18:01:26.485367694-05:00","level":"INFO","msg":"stream: closed","id":"4m3ikyrf"}

wandb/run-20250806_095353-4m3ikyrf/logs/debug.log

@@ -0,0 +1,22 @@
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Configure stats pid to 4150232
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Loading settings from /home/aye8078/.config/wandb/settings
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Loading settings from /gpfs/home/aye8078/Documents/Github/tac_vla/lerobot/wandb/settings
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/pick-blue-cube-single-arm_diff/wandb/run-20250806_095353-4m3ikyrf/logs/debug.log
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/pick-blue-cube-single-arm_diff/wandb/run-20250806_095353-4m3ikyrf/logs/debug-internal.log
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:init():830] calling init triggers
+2025-08-06 09:53:53,189 INFO    MainThread:4150232 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'a1o/pick-blue-cube_sarm', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MIN_MAX: 'MIN_MAX'>, 'ACTION': <NormalizationMode.MIN_MAX: 'MIN_MAX'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'push_to_hub': True, 'repo_id': 'a1o/diff_pick-blue-cube_sarm_policy', 'private': None, 'tags': None, 'license': None, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet18', 'crop_shape': [84, 84], 'crop_is_random': True, 'pretrained_backbone_weights': None, 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [512, 1024, 2048], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDPM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': None, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500}, 'output_dir': 'outputs/train/pick-blue-cube-single-arm_diff', 'job_name': 'diff_pick-blue-cube-single-arm', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 100000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
+2025-08-06 09:53:53,190 INFO    MainThread:4150232 [wandb_init.py:init():871] starting backend
+2025-08-06 09:53:53,911 INFO    MainThread:4150232 [wandb_init.py:init():874] sending inform_init request
+2025-08-06 09:53:53,965 INFO    MainThread:4150232 [wandb_init.py:init():882] backend started and connected
+2025-08-06 09:53:53,966 INFO    MainThread:4150232 [wandb_init.py:init():953] updated telemetry
+2025-08-06 09:53:53,966 INFO    MainThread:4150232 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-06 09:53:54,373 INFO    MainThread:4150232 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_console_start():2458] atexit reg
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_redirect():2375] Wrapping output streams.
+2025-08-06 09:53:55,933 INFO    MainThread:4150232 [wandb_run.py:_redirect():2398] Redirects installed.
+2025-08-06 09:53:55,940 INFO    MainThread:4150232 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-06 18:01:25,533 INFO    MsgRouterThr:4150232 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

wandb/run-20250806_095353-4m3ikyrf/run-4m3ikyrf.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6be5e36b76da76be0238f2c30b4700bca55b997c9e57e4f0c8116386544de427
+size 2253027