Add files using upload-large-folder tool

Abs_6D + DeltaEE/Baseline/resolved_config.yaml

@@ -0,0 +1,24 @@
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+policy:
+  type: groot
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+seed: 42
+batch_size: 128
+steps: 60000
+log_freq: 200
+output_dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/Baseline
+job_name: INSIGHT_RD_baseline
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000/pretrained_model/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000/pretrained_model/policy_postprocessor.json

@@ -0,0 +1,23 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "groot_action_unpack_unnormalize_v2",
+      "config": {
+        "env_action_dim": 10,
+        "normalize_min_max": true,
+        "clip_normalized_action": true,
+        "libero_gripper_action": false,
+        "libero_gripper_binarize": true
+      },
+      "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000/pretrained_model/train_config.json

@@ -0,0 +1,269 @@
+{
+    "dataset": {
+        "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+        "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "groot_rkd",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    16
+                ]
+            },
+            "observation.images.wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_shoulder": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.guide": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    10
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "pretrained_revision": null,
+        "chunk_size": 16,
+        "n_action_steps": 16,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "IDENTITY",
+            "ACTION": "IDENTITY"
+        },
+        "base_model_path": "nvidia/GR00T-N1.7-3B",
+        "action_decode_transform": null,
+        "embodiment_tag": "new_embodiment",
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true,
+        "tune_top_llm_layers": 0,
+        "num_inference_timesteps": null,
+        "rtc_ramp_rate": null,
+        "use_flash_attention": false,
+        "use_relative_actions": false,
+        "relative_exclude_joints": [],
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-05,
+        "warmup_steps": null,
+        "warmup_ratio": 0.05,
+        "use_bf16": true,
+        "model_params_fp32": true,
+        "image_size": [
+            256,
+            256
+        ],
+        "tokenizer_assets_repo": null,
+        "lora_rank": 0,
+        "lora_alpha": 16,
+        "lora_dropout": 0.1,
+        "lora_full_model": false,
+        "video_backend": "decord",
+        "balance_dataset_weights": true,
+        "balance_trajectory_weights": true,
+        "dataset_paths": null,
+        "output_dir": "./tmp/gr00t",
+        "save_steps": 1000,
+        "max_steps": 10000,
+        "batch_size": 32,
+        "dataloader_num_workers": 8,
+        "report_to": "wandb",
+        "resume": false,
+        "action_expert_num_layers": 8,
+        "rkd_enabled": false,
+        "rkd_vae_checkpoint": null,
+        "rkd_student_pooling": "attention",
+        "rkd_student_attention_hidden_dim": 512,
+        "rkd_student_attention_output_dim": 512,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 0.1,
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 1.0
+    },
+    "reward_model": null,
+    "output_dir": "/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline",
+    "job_name": "INSIGHT_6D_DiT8_Baseline",
+    "resume": false,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 64,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 60000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-05,
+        "grad_clip_norm": 1.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "21sn30p5",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": null
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000/resolved_config.yaml

@@ -0,0 +1,26 @@
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  action_expert_num_layers: 8
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  rkd_enabled: false
+seed: 42
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline
+job_name: INSIGHT_6D_DiT8_Baseline
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000/training_state/optimizer_param_groups.json

@@ -0,0 +1,241 @@
+[
+    {
+        "lr": 9.384074610206495e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 9.384074610206495e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200
+        ]
+    }
+]

Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000/training_state/scheduler_state.json

@@ -0,0 +1,18 @@
+{
+    "base_lrs": [
+        0.0001,
+        0.0001
+    ],
+    "last_epoch": 10000,
+    "_step_count": 10001,
+    "_is_initial": false,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        9.384074610206495e-05,
+        9.384074610206495e-05
+    ],
+    "lr_lambdas": [
+        null,
+        null
+    ]
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000/training_state/training_step.json

@@ -0,0 +1,5 @@
+{
+    "step": 10000,
+    "num_processes": 1,
+    "batch_size": 64
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model/config.json

@@ -0,0 +1,118 @@
+{
+    "type": "groot_rkd",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                16
+            ]
+        },
+        "observation.images.wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.right_shoulder": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.guide": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                10
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "pretrained_revision": null,
+    "chunk_size": 16,
+    "n_action_steps": 16,
+    "max_state_dim": 132,
+    "max_action_dim": 132,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "IDENTITY",
+        "ACTION": "IDENTITY"
+    },
+    "base_model_path": "nvidia/GR00T-N1.7-3B",
+    "action_decode_transform": null,
+    "embodiment_tag": "new_embodiment",
+    "tune_llm": false,
+    "tune_visual": false,
+    "tune_projector": true,
+    "tune_diffusion_model": true,
+    "tune_vlln": true,
+    "tune_top_llm_layers": 0,
+    "num_inference_timesteps": null,
+    "rtc_ramp_rate": null,
+    "use_flash_attention": false,
+    "use_relative_actions": false,
+    "relative_exclude_joints": [],
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-05,
+    "warmup_steps": null,
+    "warmup_ratio": 0.05,
+    "use_bf16": true,
+    "model_params_fp32": true,
+    "image_size": [
+        256,
+        256
+    ],
+    "tokenizer_assets_repo": null,
+    "lora_rank": 0,
+    "lora_alpha": 16,
+    "lora_dropout": 0.1,
+    "lora_full_model": false,
+    "video_backend": "decord",
+    "balance_dataset_weights": true,
+    "balance_trajectory_weights": true,
+    "dataset_paths": null,
+    "output_dir": "./tmp/gr00t",
+    "save_steps": 1000,
+    "max_steps": 10000,
+    "batch_size": 32,
+    "dataloader_num_workers": 8,
+    "report_to": "wandb",
+    "resume": false,
+    "action_expert_num_layers": 8,
+    "rkd_enabled": false,
+    "rkd_vae_checkpoint": null,
+    "rkd_student_pooling": "attention",
+    "rkd_student_attention_hidden_dim": 512,
+    "rkd_student_attention_output_dim": 512,
+    "rkd_fm_loss_weight": 1.0,
+    "rkd_loss_weight": 0.1,
+    "rkd_distance_weight": 1.0,
+    "rkd_angle_weight": 1.0
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model/policy_postprocessor.json

@@ -0,0 +1,23 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "groot_action_unpack_unnormalize_v2",
+      "config": {
+        "env_action_dim": 10,
+        "normalize_min_max": true,
+        "clip_normalized_action": true,
+        "libero_gripper_action": false,
+        "libero_gripper_binarize": true
+      },
+      "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model/policy_preprocessor.json

@@ -0,0 +1,78 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "groot_n1_7_pack_inputs_v1",
+      "config": {
+        "state_horizon": 1,
+        "action_horizon": 16,
+        "valid_action_horizon": 16,
+        "video_horizon": null,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "language_key": "task",
+        "formalize_language": true,
+        "embodiment_tag": "new_embodiment",
+        "embodiment_mapping": {
+          "oxe_droid_relative_eef_relative_joint": 24,
+          "xdof_relative_eef_relative_joint": 27,
+          "xdof_relative_eef_relative_joint_subtask": 27,
+          "real_g1_relative_eef_relative_joints": 25,
+          "real_r1_pro_sharpa_relative_eef": 26,
+          "real_r1_pro_sharpa_relative_eef_human": 26,
+          "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
+          "real_r1_pro_sharpa_relative_eef_mecka": 26,
+          "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
+          "simpler_env_google": 0,
+          "simpler_env_widowx": 1,
+          "libero_sim": 2,
+          "new_embodiment": 10
+        },
+        "normalize_min_max": true,
+        "state_dropout_prob": 0.0,
+        "clip_outliers": true,
+        "use_percentiles": false,
+        "video_modality_keys": null,
+        "raw_stats": null,
+        "modality_config": null
+      },
+      "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors"
+    },
+    {
+      "registry_name": "groot_n1_7_vlm_encode_v1",
+      "config": {
+        "model_name": "nvidia/Cosmos-Reason2-2B",
+        "image_crop_size": [
+          230,
+          230
+        ],
+        "image_target_size": [
+          256,
+          256
+        ],
+        "shortest_image_edge": null,
+        "crop_fraction": null,
+        "use_albumentations": false,
+        "letter_box_transform": false,
+        "device": "cuda"
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model/train_config.json

@@ -0,0 +1,269 @@
+{
+    "dataset": {
+        "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+        "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "groot_rkd",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    16
+                ]
+            },
+            "observation.images.wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_shoulder": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.guide": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    10
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "pretrained_revision": null,
+        "chunk_size": 16,
+        "n_action_steps": 16,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "IDENTITY",
+            "ACTION": "IDENTITY"
+        },
+        "base_model_path": "nvidia/GR00T-N1.7-3B",
+        "action_decode_transform": null,
+        "embodiment_tag": "new_embodiment",
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true,
+        "tune_top_llm_layers": 0,
+        "num_inference_timesteps": null,
+        "rtc_ramp_rate": null,
+        "use_flash_attention": false,
+        "use_relative_actions": false,
+        "relative_exclude_joints": [],
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-05,
+        "warmup_steps": null,
+        "warmup_ratio": 0.05,
+        "use_bf16": true,
+        "model_params_fp32": true,
+        "image_size": [
+            256,
+            256
+        ],
+        "tokenizer_assets_repo": null,
+        "lora_rank": 0,
+        "lora_alpha": 16,
+        "lora_dropout": 0.1,
+        "lora_full_model": false,
+        "video_backend": "decord",
+        "balance_dataset_weights": true,
+        "balance_trajectory_weights": true,
+        "dataset_paths": null,
+        "output_dir": "./tmp/gr00t",
+        "save_steps": 1000,
+        "max_steps": 10000,
+        "batch_size": 32,
+        "dataloader_num_workers": 8,
+        "report_to": "wandb",
+        "resume": false,
+        "action_expert_num_layers": 8,
+        "rkd_enabled": false,
+        "rkd_vae_checkpoint": null,
+        "rkd_student_pooling": "attention",
+        "rkd_student_attention_hidden_dim": 512,
+        "rkd_student_attention_output_dim": 512,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 0.1,
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 1.0
+    },
+    "reward_model": null,
+    "output_dir": "/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline",
+    "job_name": "INSIGHT_6D_DiT8_Baseline",
+    "resume": false,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 64,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 60000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-05,
+        "grad_clip_norm": 1.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "21sn30p5",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": null
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/training_state/optimizer_param_groups.json

@@ -0,0 +1,241 @@
+[
+    {
+        "lr": 5.0659979290537954e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 5.0659979290537954e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200
+        ]
+    }
+]

Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/training_state/training_step.json

@@ -0,0 +1,5 @@
+{
+    "step": 30000,
+    "num_processes": 1,
+    "batch_size": 64
+}

Abs_6D/DiT_Layer/8/Baseline/wandb/debug.log

@@ -0,0 +1,20 @@
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_setup.py:_flush():81] Configure stats pid to 814701
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_013342-21sn30p5/logs/debug.log
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_013342-21sn30p5/logs/debug-internal.log
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:init():766] calling init triggers
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline', 'job_name': 'INSIGHT_6D_DiT8_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '21sn30p5', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000', '_wandb': {}}
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:init():814] starting backend
+2026-07-23 01:33:42,527 INFO    MainThread:814701 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 01:33:42,987 INFO    MainThread:814701 [wandb_init.py:init():834] backend started and connected
+2026-07-23 01:33:42,989 INFO    MainThread:814701 [wandb_init.py:init():904] updated telemetry
+2026-07-23 01:33:42,993 INFO    MainThread:814701 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 01:33:43,715 INFO    MainThread:814701 [wandb_init.py:init():967] run resumed
+2026-07-23 01:33:43,718 INFO    MainThread:814701 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 01:33:43,821 INFO    MainThread:814701 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260722_020800-21sn30p5/files/output.log

@@ -0,0 +1,231 @@
+INFO 2026-07-22 02:08:01 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-22 02:08:01 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/21sn30p5
+INFO 2026-07-22 02:08:01 ot_train.py:298 Creating dataset
+INFO 2026-07-22 02:08:03 ot_train.py:332 Creating policy
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 7327.95it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 695/695 [00:00<00:00, 824.80it/s]
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                 | Status     |  |
+--------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.weight       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.bias     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.bias         | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+INFO 2026-07-22 02:08:09 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-22 02:08:09 ot_train.py:437 Output dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline
+INFO 2026-07-22 02:08:09 ot_train.py:444 cfg.steps=60000 (60K)
+INFO 2026-07-22 02:08:09 ot_train.py:445 dataset.num_frames=218367 (218K)
+INFO 2026-07-22 02:08:09 ot_train.py:446 dataset.num_episodes=4930
+INFO 2026-07-22 02:08:09 ot_train.py:449 Effective batch size: 64 x 1 = 64
+INFO 2026-07-22 02:08:09 ot_train.py:450 num_learnable_params=808512640 (809M)
+INFO 2026-07-22 02:08:09 ot_train.py:451 num_total_params=2333587585 (2B)
+Training:   0%|          | 0/60000 [00:00<?, ?step/s]INFO 2026-07-22 02:08:09 ot_train.py:602 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   0%|          | 200/60000 [04:14<21:11:56,  1.28s/step]INFO 2026-07-22 02:12:23 ot_train.py:646 step:200 smpl:13K ep:289 epch:0.06 loss:0.998 grdn:1.416 lr:2.0e-05 updt_s:0.697 data_s:0.572 smp/s:50 mem_gb:24.86
+Training:   1%|          | 400/60000 [08:24<21:24:34,  1.29s/step]INFO 2026-07-22 02:16:34 ot_train.py:646 step:400 smpl:26K ep:578 epch:0.12 loss:0.315 grdn:1.801 lr:6.0e-05 updt_s:0.680 data_s:0.572 smp/s:51 mem_gb:24.88
+Training:   1%|          | 600/60000 [12:33<21:32:59,  1.31s/step]INFO 2026-07-22 02:20:42 ot_train.py:646 step:600 smpl:38K ep:867 epch:0.18 loss:0.229 grdn:1.092 lr:9.5e-05 updt_s:0.678 data_s:0.563 smp/s:52 mem_gb:24.88
+Training:   1%|▏         | 800/60000 [16:41<20:49:44,  1.27s/step]INFO 2026-07-22 02:24:51 ot_train.py:646 step:800 smpl:51K ep:1K epch:0.23 loss:0.182 grdn:0.809 lr:1.0e-04 updt_s:0.676 data_s:0.567 smp/s:51 mem_gb:24.88
+Training:   2%|▏         | 1000/60000 [20:51<20:28:48,  1.25s/step]INFO 2026-07-22 02:29:00 ot_train.py:646 step:1K smpl:64K ep:1K epch:0.29 loss:0.158 grdn:0.698 lr:1.0e-04 updt_s:0.680 data_s:0.564 smp/s:51 mem_gb:24.88
+Training:   2%|▏         | 1200/60000 [24:59<20:04:58,  1.23s/step]INFO 2026-07-22 02:33:08 ot_train.py:646 step:1K smpl:77K ep:2K epch:0.35 loss:0.145 grdn:0.644 lr:1.0e-04 updt_s:0.676 data_s:0.564 smp/s:52 mem_gb:24.88
+Training:   2%|▏         | 1400/60000 [29:08<20:01:36,  1.23s/step]INFO 2026-07-22 02:37:17 ot_train.py:646 step:1K smpl:90K ep:2K epch:0.41 loss:0.134 grdn:0.607 lr:1.0e-04 updt_s:0.677 data_s:0.566 smp/s:51 mem_gb:24.88
+Training:   3%|▎         | 1600/60000 [33:16<19:54:58,  1.23s/step]INFO 2026-07-22 02:41:25 ot_train.py:646 step:2K smpl:102K ep:2K epch:0.47 loss:0.127 grdn:0.569 lr:1.0e-04 updt_s:0.675 data_s:0.563 smp/s:52 mem_gb:24.88
+Training:   3%|▎         | 1800/60000 [37:25<21:26:17,  1.33s/step]INFO 2026-07-22 02:45:34 ot_train.py:646 step:2K smpl:115K ep:3K epch:0.53 loss:0.122 grdn:0.533 lr:1.0e-04 updt_s:0.680 data_s:0.565 smp/s:51 mem_gb:24.88
+Training:   3%|▎         | 2000/60000 [41:42<20:16:58,  1.26s/step]INFO 2026-07-22 02:49:51 ot_train.py:646 step:2K smpl:128K ep:3K epch:0.59 loss:0.116 grdn:0.513 lr:1.0e-04 updt_s:0.689 data_s:0.593 smp/s:50 mem_gb:24.88
+Training:   4%|▎         | 2200/60000 [46:03<20:39:54,  1.29s/step]INFO 2026-07-22 02:54:13 ot_train.py:646 step:2K smpl:141K ep:3K epch:0.64 loss:0.112 grdn:0.494 lr:1.0e-04 updt_s:0.717 data_s:0.590 smp/s:49 mem_gb:24.88
+Training:   4%|▍         | 2400/60000 [50:19<20:15:54,  1.27s/step]INFO 2026-07-22 02:58:28 ot_train.py:646 step:2K smpl:154K ep:3K epch:0.70 loss:0.111 grdn:0.494 lr:1.0e-04 updt_s:0.700 data_s:0.578 smp/s:50 mem_gb:24.86
+Training:   4%|▍         | 2600/60000 [54:27<20:09:02,  1.26s/step]INFO 2026-07-22 03:02:36 ot_train.py:646 step:3K smpl:166K ep:4K epch:0.76 loss:0.106 grdn:0.458 lr:1.0e-04 updt_s:0.677 data_s:0.561 smp/s:52 mem_gb:24.88
+Training:   5%|▍         | 2800/60000 [58:44<25:30:01,  1.60s/step]INFO 2026-07-22 03:06:54 ot_train.py:646 step:3K smpl:179K ep:4K epch:0.82 loss:0.105 grdn:0.454 lr:1.0e-04 updt_s:0.720 data_s:0.567 smp/s:50 mem_gb:24.88
+Training:   5%|▌         | 3000/60000 [1:02:57<19:20:08,  1.22s/step]INFO 2026-07-22 03:11:06 ot_train.py:646 step:3K smpl:192K ep:4K epch:0.88 loss:0.101 grdn:0.446 lr:1.0e-04 updt_s:0.700 data_s:0.560 smp/s:51 mem_gb:24.88
+Training:   5%|▌         | 3200/60000 [1:07:10<19:46:37,  1.25s/step]INFO 2026-07-22 03:15:19 ot_train.py:646 step:3K smpl:205K ep:5K epch:0.94 loss:0.101 grdn:0.440 lr:1.0e-04 updt_s:0.704 data_s:0.562 smp/s:51 mem_gb:24.88
+Training:   6%|▌         | 3400/60000 [1:11:19<20:13:27,  1.29s/step]INFO 2026-07-22 03:19:28 ot_train.py:646 step:3K smpl:218K ep:5K epch:1.00 loss:0.098 grdn:0.421 lr:9.9e-05 updt_s:0.677 data_s:0.565 smp/s:52 mem_gb:24.88
+Training:   6%|▌         | 3600/60000 [1:16:26<19:35:52,  1.25s/step]INFO 2026-07-22 03:24:35 ot_train.py:646 step:4K smpl:230K ep:5K epch:1.06 loss:0.099 grdn:0.438 lr:9.9e-05 updt_s:0.932 data_s:0.603 smp/s:42 mem_gb:24.88
+Training:   6%|▋         | 3800/60000 [1:20:32<18:55:33,  1.21s/step]INFO 2026-07-22 03:28:42 ot_train.py:646 step:4K smpl:243K ep:5K epch:1.11 loss:0.097 grdn:0.413 lr:9.9e-05 updt_s:0.675 data_s:0.556 smp/s:52 mem_gb:24.88
+Training:   7%|▋         | 4000/60000 [1:24:58<19:26:31,  1.25s/step]INFO 2026-07-22 03:33:08 ot_train.py:646 step:4K smpl:256K ep:6K epch:1.17 loss:0.097 grdn:0.424 lr:9.9e-05 updt_s:0.752 data_s:0.576 smp/s:48 mem_gb:24.88
+Training:   7%|▋         | 4200/60000 [1:29:09<18:58:07,  1.22s/step]INFO 2026-07-22 03:37:18 ot_train.py:646 step:4K smpl:269K ep:6K epch:1.23 loss:0.093 grdn:0.397 lr:9.9e-05 updt_s:0.685 data_s:0.567 smp/s:51 mem_gb:24.88
+Training:   7%|▋         | 4400/60000 [1:33:18<18:58:27,  1.23s/step]INFO 2026-07-22 03:41:27 ot_train.py:646 step:4K smpl:282K ep:6K epch:1.29 loss:0.092 grdn:0.396 lr:9.9e-05 updt_s:0.678 data_s:0.566 smp/s:51 mem_gb:24.88
+Training:   8%|▊         | 4600/60000 [1:37:29<20:44:42,  1.35s/step]INFO 2026-07-22 03:45:38 ot_train.py:646 step:5K smpl:294K ep:7K epch:1.35 loss:0.091 grdn:0.391 lr:9.9e-05 updt_s:0.682 data_s:0.572 smp/s:51 mem_gb:24.86
+Training:   8%|▊         | 4800/60000 [1:41:38<19:24:01,  1.27s/step]INFO 2026-07-22 03:49:48 ot_train.py:646 step:5K smpl:307K ep:7K epch:1.41 loss:0.089 grdn:0.384 lr:9.9e-05 updt_s:0.679 data_s:0.567 smp/s:51 mem_gb:24.88
+Training:   8%|▊         | 5000/60000 [1:45:48<18:49:33,  1.23s/step]INFO 2026-07-22 03:53:57 ot_train.py:646 step:5K smpl:320K ep:7K epch:1.47 loss:0.089 grdn:0.388 lr:9.9e-05 updt_s:0.682 data_s:0.565 smp/s:51 mem_gb:24.88
+Training:   9%|▊         | 5200/60000 [1:49:58<19:21:44,  1.27s/step]INFO 2026-07-22 03:58:07 ot_train.py:646 step:5K smpl:333K ep:8K epch:1.52 loss:0.089 grdn:0.382 lr:9.9e-05 updt_s:0.679 data_s:0.570 smp/s:51 mem_gb:24.88
+Training:   9%|▉         | 5400/60000 [1:54:09<18:45:50,  1.24s/step]INFO 2026-07-22 04:02:18 ot_train.py:646 step:5K smpl:346K ep:8K epch:1.58 loss:0.087 grdn:0.378 lr:9.8e-05 updt_s:0.686 data_s:0.568 smp/s:51 mem_gb:24.88
+Training:   9%|▉         | 5600/60000 [1:58:18<18:35:47,  1.23s/step]INFO 2026-07-22 04:06:28 ot_train.py:646 step:6K smpl:358K ep:8K epch:1.64 loss:0.085 grdn:0.371 lr:9.8e-05 updt_s:0.679 data_s:0.566 smp/s:51 mem_gb:24.88
+Training:  10%|▉         | 5800/60000 [2:02:27<18:19:49,  1.22s/step]INFO 2026-07-22 04:10:36 ot_train.py:646 step:6K smpl:371K ep:8K epch:1.70 loss:0.085 grdn:0.375 lr:9.8e-05 updt_s:0.683 data_s:0.560 smp/s:51 mem_gb:24.88
+Training:  10%|█         | 6000/60000 [2:06:37<18:32:42,  1.24s/step]INFO 2026-07-22 04:14:46 ot_train.py:646 step:6K smpl:384K ep:9K epch:1.76 loss:0.085 grdn:0.374 lr:9.8e-05 updt_s:0.680 data_s:0.567 smp/s:51 mem_gb:24.88
+Training:  10%|█         | 6200/60000 [2:10:45<18:24:04,  1.23s/step]INFO 2026-07-22 04:18:54 ot_train.py:646 step:6K smpl:397K ep:9K epch:1.82 loss:0.086 grdn:0.370 lr:9.8e-05 updt_s:0.670 data_s:0.569 smp/s:52 mem_gb:24.88
+Training:  11%|█         | 6400/60000 [2:14:59<18:33:11,  1.25s/step]INFO 2026-07-22 04:23:08 ot_train.py:646 step:6K smpl:410K ep:9K epch:1.88 loss:0.084 grdn:0.369 lr:9.8e-05 updt_s:0.699 data_s:0.570 smp/s:50 mem_gb:24.88
+Training:  11%|█         | 6600/60000 [2:19:19<18:30:20,  1.25s/step]INFO 2026-07-22 04:27:28 ot_train.py:646 step:7K smpl:422K ep:10K epch:1.93 loss:0.084 grdn:0.371 lr:9.8e-05 updt_s:0.702 data_s:0.597 smp/s:49 mem_gb:24.88
+Training:  11%|█▏        | 6800/60000 [2:23:38<19:19:14,  1.31s/step]INFO 2026-07-22 04:31:47 ot_train.py:646 step:7K smpl:435K ep:10K epch:1.99 loss:0.084 grdn:0.372 lr:9.7e-05 updt_s:0.693 data_s:0.602 smp/s:49 mem_gb:24.86
+Training:  12%|█▏        | 7000/60000 [2:27:54<18:28:29,  1.25s/step]INFO 2026-07-22 04:36:03 ot_train.py:646 step:7K smpl:448K ep:10K epch:2.05 loss:0.082 grdn:0.357 lr:9.7e-05 updt_s:0.695 data_s:0.584 smp/s:50 mem_gb:24.88
+Training:  12%|█▏        | 7200/60000 [2:32:29<21:37:48,  1.47s/step]INFO 2026-07-22 04:40:39 ot_train.py:646 step:7K smpl:461K ep:10K epch:2.11 loss:0.080 grdn:0.341 lr:9.7e-05 updt_s:0.720 data_s:0.655 smp/s:47 mem_gb:24.88
+Training:  12%|█▏        | 7400/60000 [2:37:38<23:24:55,  1.60s/step]INFO 2026-07-22 04:45:47 ot_train.py:646 step:7K smpl:474K ep:11K epch:2.17 loss:0.078 grdn:0.341 lr:9.7e-05 updt_s:0.806 data_s:0.737 smp/s:41 mem_gb:24.88
+Training:  13%|█▎        | 7600/60000 [2:42:52<21:01:02,  1.44s/step]INFO 2026-07-22 04:51:02 ot_train.py:646 step:8K smpl:486K ep:11K epch:2.23 loss:0.082 grdn:0.350 lr:9.7e-05 updt_s:0.818 data_s:0.751 smp/s:41 mem_gb:24.88
+Training:  13%|█▎        | 7800/60000 [2:48:06<22:22:47,  1.54s/step]INFO 2026-07-22 04:56:15 ot_train.py:646 step:8K smpl:499K ep:11K epch:2.29 loss:0.080 grdn:0.338 lr:9.6e-05 updt_s:0.760 data_s:0.805 smp/s:41 mem_gb:24.88
+Training:  13%|█▎        | 8000/60000 [2:53:55<21:53:35,  1.52s/step]INFO 2026-07-22 05:02:05 ot_train.py:646 step:8K smpl:512K ep:12K epch:2.34 loss:0.078 grdn:0.363 lr:9.6e-05 updt_s:0.746 data_s:1.002 smp/s:37 mem_gb:24.88
+Training:  14%|█▎        | 8200/60000 [3:00:13<26:36:53,  1.85s/step]INFO 2026-07-22 05:08:22 ot_train.py:646 step:8K smpl:525K ep:12K epch:2.40 loss:0.079 grdn:0.353 lr:9.6e-05 updt_s:0.781 data_s:1.105 smp/s:34 mem_gb:24.88
+Training:  14%|█▍        | 8400/60000 [3:06:20<22:59:19,  1.60s/step]INFO 2026-07-22 05:14:30 ot_train.py:646 step:8K smpl:538K ep:12K epch:2.46 loss:0.079 grdn:0.341 lr:9.6e-05 updt_s:0.754 data_s:1.082 smp/s:35 mem_gb:24.88
+Training:  14%|█▍        | 8600/60000 [3:12:30<29:57:33,  2.10s/step]INFO 2026-07-22 05:20:39 ot_train.py:646 step:9K smpl:550K ep:12K epch:2.52 loss:0.079 grdn:0.350 lr:9.6e-05 updt_s:0.768 data_s:1.077 smp/s:35 mem_gb:24.88
+Training:  15%|█▍        | 8800/60000 [3:17:12<17:47:07,  1.25s/step]INFO 2026-07-22 05:25:22 ot_train.py:646 step:9K smpl:563K ep:13K epch:2.58 loss:0.079 grdn:0.368 lr:9.5e-05 updt_s:0.693 data_s:0.718 smp/s:45 mem_gb:24.87
+Training:  15%|█▌        | 9000/60000 [3:21:23<18:02:52,  1.27s/step]INFO 2026-07-22 05:29:33 ot_train.py:646 step:9K smpl:576K ep:13K epch:2.64 loss:0.077 grdn:0.336 lr:9.5e-05 updt_s:0.714 data_s:0.540 smp/s:51 mem_gb:24.88
+Training:  15%|█▌        | 9200/60000 [3:26:27<22:53:21,  1.62s/step]INFO 2026-07-22 05:34:37 ot_train.py:646 step:9K smpl:589K ep:13K epch:2.70 loss:0.076 grdn:0.328 lr:9.5e-05 updt_s:0.773 data_s:0.745 smp/s:42 mem_gb:24.86
+Training:  16%|█▌        | 9400/60000 [3:31:33<21:26:44,  1.53s/step]INFO 2026-07-22 05:39:42 ot_train.py:646 step:9K smpl:602K ep:14K epch:2.75 loss:0.076 grdn:0.338 lr:9.5e-05 updt_s:0.781 data_s:0.744 smp/s:42 mem_gb:24.88
+Training:  16%|█▌        | 9600/60000 [3:36:38<21:53:41,  1.56s/step]INFO 2026-07-22 05:44:48 ot_train.py:646 step:10K smpl:614K ep:14K epch:2.81 loss:0.075 grdn:0.324 lr:9.4e-05 updt_s:0.779 data_s:0.748 smp/s:42 mem_gb:24.88
+Training:  16%|█▋        | 9800/60000 [3:42:22<22:22:02,  1.60s/step]INFO 2026-07-22 05:50:31 ot_train.py:646 step:10K smpl:627K ep:14K epch:2.87 loss:0.075 grdn:0.355 lr:9.4e-05 updt_s:0.756 data_s:0.960 smp/s:37 mem_gb:24.88
+Training:  17%|█▋        | 10000/60000 [3:47:46<32:39:13,  2.35s/step]INFO 2026-07-22 05:55:55 ot_train.py:646 step:10K smpl:640K ep:14K epch:2.93 loss:0.075 grdn:0.337 lr:9.4e-05 updt_s:0.773 data_s:0.847 smp/s:40 mem_gb:24.88
+INFO 2026-07-22 05:55:55 ot_train.py:691 Checkpoint policy after step 10000
+INFO 2026-07-22 05:56:28 in_yaml.py:1270 Saved config, action-space, prompt, and phase provenance in /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline and /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/checkpoints/010000
+Training:  17%|█▋        | 10200/60000 [3:55:05<20:20:07,  1.47s/step]INFO 2026-07-22 06:03:14 ot_train.py:646 step:10K smpl:653K ep:15K epch:2.99 loss:0.074 grdn:0.343 lr:9.4e-05 updt_s:0.756 data_s:1.275 smp/s:32 mem_gb:24.88
+Training:  17%|█▋        | 10400/60000 [4:00:45<24:23:20,  1.77s/step]INFO 2026-07-22 06:08:54 ot_train.py:646 step:10K smpl:666K ep:15K epch:3.05 loss:0.072 grdn:0.326 lr:9.3e-05 updt_s:0.773 data_s:0.926 smp/s:38 mem_gb:24.88
+Training:  18%|█▊        | 10600/60000 [4:06:14<21:35:25,  1.57s/step]INFO 2026-07-22 06:14:23 ot_train.py:646 step:11K smpl:678K ep:15K epch:3.11 loss:0.072 grdn:0.314 lr:9.3e-05 updt_s:0.780 data_s:0.865 smp/s:39 mem_gb:24.88
+Training:  18%|█▊        | 10800/60000 [4:11:29<23:55:18,  1.75s/step]INFO 2026-07-22 06:19:38 ot_train.py:646 step:11K smpl:691K ep:16K epch:3.17 loss:0.071 grdn:0.321 lr:9.3e-05 updt_s:0.818 data_s:0.754 smp/s:41 mem_gb:24.88
+Training:  18%|█▊        | 11000/60000 [4:16:49<23:03:08,  1.69s/step]INFO 2026-07-22 06:24:58 ot_train.py:646 step:11K smpl:704K ep:16K epch:3.22 loss:0.071 grdn:0.315 lr:9.3e-05 updt_s:0.799 data_s:0.800 smp/s:40 mem_gb:24.88
+Training:  19%|█▊        | 11200/60000 [4:22:43<22:54:57,  1.69s/step]INFO 2026-07-22 06:30:52 ot_train.py:646 step:11K smpl:717K ep:16K epch:3.28 loss:0.071 grdn:0.328 lr:9.2e-05 updt_s:0.798 data_s:0.969 smp/s:36 mem_gb:24.88
+Training:  19%|█▉        | 11400/60000 [4:28:27<23:45:49,  1.76s/step]INFO 2026-07-22 06:36:37 ot_train.py:646 step:11K smpl:730K ep:16K epch:3.34 loss:0.071 grdn:0.324 lr:9.2e-05 updt_s:0.761 data_s:0.962 smp/s:37 mem_gb:24.86
+Training:  19%|█▉        | 11600/60000 [4:34:42<21:32:20,  1.60s/step]INFO 2026-07-22 06:42:52 ot_train.py:646 step:12K smpl:742K ep:17K epch:3.40 loss:0.071 grdn:0.326 lr:9.2e-05 updt_s:0.872 data_s:1.002 smp/s:34 mem_gb:24.88
+Training:  20%|█▉        | 11800/60000 [4:40:19<20:46:08,  1.55s/step]INFO 2026-07-22 06:48:29 ot_train.py:646 step:12K smpl:755K ep:17K epch:3.46 loss:0.071 grdn:0.325 lr:9.2e-05 updt_s:0.826 data_s:0.857 smp/s:38 mem_gb:24.88
+Training:  20%|██        | 12000/60000 [4:45:52<22:37:29,  1.70s/step]INFO 2026-07-22 06:54:01 ot_train.py:646 step:12K smpl:768K ep:17K epch:3.52 loss:0.070 grdn:0.315 lr:9.1e-05 updt_s:0.818 data_s:0.843 smp/s:39 mem_gb:24.88
+Training:  20%|██        | 12200/60000 [4:51:12<21:25:54,  1.61s/step]INFO 2026-07-22 06:59:21 ot_train.py:646 step:12K smpl:781K ep:18K epch:3.58 loss:0.071 grdn:0.332 lr:9.1e-05 updt_s:0.836 data_s:0.761 smp/s:40 mem_gb:24.88
+Training:  21%|██        | 12400/60000 [4:57:15<28:09:50,  2.13s/step]INFO 2026-07-22 07:05:24 ot_train.py:646 step:12K smpl:794K ep:18K epch:3.63 loss:0.066 grdn:0.310 lr:9.1e-05 updt_s:0.796 data_s:1.016 smp/s:35 mem_gb:24.88
+Training:  21%|██        | 12600/60000 [5:03:48<21:34:55,  1.64s/step]INFO 2026-07-22 07:11:57 ot_train.py:646 step:13K smpl:806K ep:18K epch:3.69 loss:0.067 grdn:0.311 lr:9.0e-05 updt_s:0.796 data_s:1.168 smp/s:33 mem_gb:24.88
+Training:  21%|██▏       | 12800/60000 [5:09:01<17:55:21,  1.37s/step]INFO 2026-07-22 07:17:11 ot_train.py:646 step:13K smpl:819K ep:18K epch:3.75 loss:0.069 grdn:0.328 lr:9.0e-05 updt_s:0.806 data_s:0.761 smp/s:41 mem_gb:24.88
+Training:  22%|██▏       | 13000/60000 [5:13:28<16:58:26,  1.30s/step]INFO 2026-07-22 07:21:37 ot_train.py:646 step:13K smpl:832K ep:19K epch:3.81 loss:0.067 grdn:0.322 lr:9.0e-05 updt_s:0.721 data_s:0.611 smp/s:48 mem_gb:24.88
+Training:  22%|██▏       | 13200/60000 [5:18:07<20:37:49,  1.59s/step]INFO 2026-07-22 07:26:16 ot_train.py:646 step:13K smpl:845K ep:19K epch:3.87 loss:0.068 grdn:0.312 lr:8.9e-05 updt_s:0.742 data_s:0.650 smp/s:46 mem_gb:24.88
+Training:  22%|██▏       | 13400/60000 [5:23:01<19:10:58,  1.48s/step]INFO 2026-07-22 07:31:10 ot_train.py:646 step:13K smpl:858K ep:19K epch:3.93 loss:0.069 grdn:0.344 lr:8.9e-05 updt_s:0.755 data_s:0.715 smp/s:44 mem_gb:24.88
+Training:  23%|██▎       | 13600/60000 [5:27:33<16:20:55,  1.27s/step]INFO 2026-07-22 07:35:42 ot_train.py:646 step:14K smpl:870K ep:20K epch:3.99 loss:0.066 grdn:0.305 lr:8.9e-05 updt_s:0.721 data_s:0.636 smp/s:47 mem_gb:24.86
+Training:  23%|██▎       | 13800/60000 [5:31:53<17:09:20,  1.34s/step]INFO 2026-07-22 07:40:02 ot_train.py:646 step:14K smpl:883K ep:20K epch:4.04 loss:0.065 grdn:0.298 lr:8.8e-05 updt_s:0.697 data_s:0.602 smp/s:49 mem_gb:24.88
+Training:  23%|██▎       | 14000/60000 [5:36:12<16:42:34,  1.31s/step]INFO 2026-07-22 07:44:22 ot_train.py:646 step:14K smpl:896K ep:20K epch:4.10 loss:0.065 grdn:0.311 lr:8.8e-05 updt_s:0.726 data_s:0.571 smp/s:49 mem_gb:24.88
+Training:  24%|██▎       | 14200/60000 [5:40:29<16:00:21,  1.26s/step]INFO 2026-07-22 07:48:38 ot_train.py:646 step:14K smpl:909K ep:21K epch:4.16 loss:0.066 grdn:0.316 lr:8.8e-05 updt_s:0.727 data_s:0.553 smp/s:50 mem_gb:24.88
+Training:  24%|██▍       | 14400/60000 [5:44:41<15:52:49,  1.25s/step]INFO 2026-07-22 07:52:50 ot_train.py:646 step:14K smpl:922K ep:21K epch:4.22 loss:0.064 grdn:0.302 lr:8.7e-05 updt_s:0.711 data_s:0.548 smp/s:51 mem_gb:24.88
+Training:  24%|██▍       | 14600/60000 [5:48:55<15:46:51,  1.25s/step]INFO 2026-07-22 07:57:04 ot_train.py:646 step:15K smpl:934K ep:21K epch:4.28 loss:0.064 grdn:0.296 lr:8.7e-05 updt_s:0.718 data_s:0.551 smp/s:50 mem_gb:24.88
+Training:  25%|██▍       | 14800/60000 [5:53:10<15:56:25,  1.27s/step]INFO 2026-07-22 08:01:19 ot_train.py:646 step:15K smpl:947K ep:21K epch:4.34 loss:0.066 grdn:0.307 lr:8.7e-05 updt_s:0.715 data_s:0.558 smp/s:50 mem_gb:24.87
+Training:  25%|██▌       | 15000/60000 [5:57:24<15:41:01,  1.25s/step]INFO 2026-07-22 08:05:34 ot_train.py:646 step:15K smpl:960K ep:22K epch:4.40 loss:0.066 grdn:0.326 lr:8.6e-05 updt_s:0.719 data_s:0.553 smp/s:50 mem_gb:24.88
+Training:  25%|██▌       | 15200/60000 [6:01:40<15:33:04,  1.25s/step]INFO 2026-07-22 08:09:50 ot_train.py:646 step:15K smpl:973K ep:22K epch:4.45 loss:0.066 grdn:0.320 lr:8.6e-05 updt_s:0.715 data_s:0.565 smp/s:50 mem_gb:24.88
+Training:  26%|██▌       | 15400/60000 [6:05:54<16:21:02,  1.32s/step]INFO 2026-07-22 08:14:03 ot_train.py:646 step:15K smpl:986K ep:22K epch:4.51 loss:0.065 grdn:0.317 lr:8.5e-05 updt_s:0.703 data_s:0.561 smp/s:51 mem_gb:24.88
+Training:  26%|██▌       | 15600/60000 [6:10:02<15:55:40,  1.29s/step]INFO 2026-07-22 08:18:11 ot_train.py:646 step:16K smpl:998K ep:23K epch:4.57 loss:0.064 grdn:0.299 lr:8.5e-05 updt_s:0.670 data_s:0.571 smp/s:52 mem_gb:24.88
+Training:  26%|██▋       | 15800/60000 [6:14:24<19:33:31,  1.59s/step]INFO 2026-07-22 08:22:34 ot_train.py:646 step:16K smpl:1M ep:23K epch:4.63 loss:0.063 grdn:0.285 lr:8.5e-05 updt_s:0.747 data_s:0.564 smp/s:49 mem_gb:24.86
+Training:  27%|██▋       | 16000/60000 [6:18:46<16:07:30,  1.32s/step]INFO 2026-07-22 08:26:55 ot_train.py:646 step:16K smpl:1M ep:23K epch:4.69 loss:0.062 grdn:0.313 lr:8.4e-05 updt_s:0.751 data_s:0.555 smp/s:49 mem_gb:24.88
+Training:  27%|██▋       | 16200/60000 [6:23:20<18:23:48,  1.51s/step]INFO 2026-07-22 08:31:30 ot_train.py:646 step:16K smpl:1M ep:23K epch:4.75 loss:0.063 grdn:0.306 lr:8.4e-05 updt_s:0.817 data_s:0.555 smp/s:47 mem_gb:24.88
+Training:  27%|██▋       | 16400/60000 [6:27:55<17:01:37,  1.41s/step]INFO 2026-07-22 08:36:04 ot_train.py:646 step:16K smpl:1M ep:24K epch:4.81 loss:0.064 grdn:0.311 lr:8.4e-05 updt_s:0.803 data_s:0.566 smp/s:47 mem_gb:24.88
+Training:  28%|██▊       | 16600/60000 [6:32:30<18:03:20,  1.50s/step]INFO 2026-07-22 08:40:39 ot_train.py:646 step:17K smpl:1M ep:24K epch:4.87 loss:0.062 grdn:0.314 lr:8.3e-05 updt_s:0.802 data_s:0.573 smp/s:47 mem_gb:24.88
+Training:  28%|██▊       | 16800/60000 [6:36:52<17:04:46,  1.42s/step]INFO 2026-07-22 08:45:02 ot_train.py:646 step:17K smpl:1M ep:24K epch:4.92 loss:0.062 grdn:0.308 lr:8.3e-05 updt_s:0.747 data_s:0.564 smp/s:49 mem_gb:24.88
+Training:  28%|██▊       | 17000/60000 [6:41:23<17:41:22,  1.48s/step]INFO 2026-07-22 08:49:33 ot_train.py:646 step:17K smpl:1M ep:25K epch:4.98 loss:0.062 grdn:0.299 lr:8.2e-05 updt_s:0.791 data_s:0.561 smp/s:47 mem_gb:24.88
+Training:  29%|██▊       | 17200/60000 [6:45:43<16:50:33,  1.42s/step]INFO 2026-07-22 08:53:52 ot_train.py:646 step:17K smpl:1M ep:25K epch:5.04 loss:0.060 grdn:0.301 lr:8.2e-05 updt_s:0.728 data_s:0.568 smp/s:49 mem_gb:24.88
+Training:  29%|██▉       | 17400/60000 [6:50:02<16:07:28,  1.36s/step]INFO 2026-07-22 08:58:12 ot_train.py:646 step:17K smpl:1M ep:25K epch:5.10 loss:0.061 grdn:0.301 lr:8.2e-05 updt_s:0.734 data_s:0.563 smp/s:49 mem_gb:24.88
+Training:  29%|██▉       | 17600/60000 [6:54:37<16:17:11,  1.38s/step]INFO 2026-07-22 09:02:46 ot_train.py:646 step:18K smpl:1M ep:25K epch:5.16 loss:0.063 grdn:0.319 lr:8.1e-05 updt_s:0.812 data_s:0.559 smp/s:47 mem_gb:24.88
+Training:  30%|██▉       | 17800/60000 [6:59:07<17:39:57,  1.51s/step]INFO 2026-07-22 09:07:17 ot_train.py:646 step:18K smpl:1M ep:26K epch:5.22 loss:0.060 grdn:0.292 lr:8.1e-05 updt_s:0.777 data_s:0.576 smp/s:47 mem_gb:24.88
+Training:  30%|███       | 18000/60000 [7:03:37<14:33:00,  1.25s/step]INFO 2026-07-22 09:11:46 ot_train.py:646 step:18K smpl:1M ep:26K epch:5.28 loss:0.061 grdn:0.298 lr:8.0e-05 updt_s:0.780 data_s:0.567 smp/s:48 mem_gb:24.88
+Training:  30%|███       | 18200/60000 [7:08:05<16:49:00,  1.45s/step]INFO 2026-07-22 09:16:14 ot_train.py:646 step:18K smpl:1M ep:26K epch:5.33 loss:0.059 grdn:0.296 lr:8.0e-05 updt_s:0.782 data_s:0.556 smp/s:48 mem_gb:24.86
+Training:  31%|███       | 18400/60000 [7:12:40<15:12:30,  1.32s/step]INFO 2026-07-22 09:20:49 ot_train.py:646 step:18K smpl:1M ep:27K epch:5.39 loss:0.060 grdn:0.301 lr:7.9e-05 updt_s:0.823 data_s:0.550 smp/s:47 mem_gb:24.88
+Training:  31%|███       | 18600/60000 [7:17:20<15:49:10,  1.38s/step]INFO 2026-07-22 09:25:29 ot_train.py:646 step:19K smpl:1M ep:27K epch:5.45 loss:0.059 grdn:0.292 lr:7.9e-05 updt_s:0.838 data_s:0.563 smp/s:46 mem_gb:24.88
+Training:  31%|███▏      | 18800/60000 [7:21:32<15:10:38,  1.33s/step]INFO 2026-07-22 09:29:42 ot_train.py:646 step:19K smpl:1M ep:27K epch:5.51 loss:0.059 grdn:0.287 lr:7.9e-05 updt_s:0.693 data_s:0.568 smp/s:51 mem_gb:24.88
+Training:  32%|███▏      | 19000/60000 [7:26:03<16:07:17,  1.42s/step]INFO 2026-07-22 09:34:12 ot_train.py:646 step:19K smpl:1M ep:27K epch:5.57 loss:0.058 grdn:0.285 lr:7.8e-05 updt_s:0.789 data_s:0.562 smp/s:47 mem_gb:24.88
+Training:  32%|███▏      | 19200/60000 [7:30:30<15:55:23,  1.40s/step]INFO 2026-07-22 09:38:39 ot_train.py:646 step:19K smpl:1M ep:28K epch:5.63 loss:0.060 grdn:0.306 lr:7.8e-05 updt_s:0.774 data_s:0.560 smp/s:48 mem_gb:24.88
+Training:  32%|███▏      | 19400/60000 [7:34:54<14:40:34,  1.30s/step]INFO 2026-07-22 09:43:03 ot_train.py:646 step:19K smpl:1M ep:28K epch:5.69 loss:0.059 grdn:0.295 lr:7.7e-05 updt_s:0.766 data_s:0.554 smp/s:48 mem_gb:24.88
+Training:  33%|███▎      | 19600/60000 [7:39:34<16:53:21,  1.50s/step]INFO 2026-07-22 09:47:44 ot_train.py:646 step:20K smpl:1M ep:28K epch:5.74 loss:0.059 grdn:0.297 lr:7.7e-05 updt_s:0.836 data_s:0.564 smp/s:46 mem_gb:24.88
+Training:  33%|███▎      | 19800/60000 [7:44:08<13:35:51,  1.22s/step]INFO 2026-07-22 09:52:18 ot_train.py:646 step:20K smpl:1M ep:29K epch:5.80 loss:0.058 grdn:0.296 lr:7.6e-05 updt_s:0.812 data_s:0.556 smp/s:47 mem_gb:24.88
+Training:  33%|███▎      | 20000/60000 [7:48:31<14:58:08,  1.35s/step]INFO 2026-07-22 09:56:41 ot_train.py:646 step:20K smpl:1M ep:29K epch:5.86 loss:0.058 grdn:0.292 lr:7.6e-05 updt_s:0.764 data_s:0.552 smp/s:49 mem_gb:24.88
+Training:  34%|███▎      | 20200/60000 [7:53:02<16:05:27,  1.46s/step]INFO 2026-07-22 10:01:11 ot_train.py:646 step:20K smpl:1M ep:29K epch:5.92 loss:0.058 grdn:0.289 lr:7.6e-05 updt_s:0.784 data_s:0.565 smp/s:47 mem_gb:24.88
+Training:  34%|███▍      | 20400/60000 [7:57:26<16:04:48,  1.46s/step]INFO 2026-07-22 10:05:35 ot_train.py:646 step:20K smpl:1M ep:29K epch:5.98 loss:0.055 grdn:0.291 lr:7.5e-05 updt_s:0.754 data_s:0.565 smp/s:49 mem_gb:24.86
+Training:  34%|███▍      | 20600/60000 [8:01:44<13:45:53,  1.26s/step]INFO 2026-07-22 10:09:53 ot_train.py:646 step:21K smpl:1M ep:30K epch:6.04 loss:0.056 grdn:0.288 lr:7.5e-05 updt_s:0.739 data_s:0.551 smp/s:50 mem_gb:24.88
+Training:  35%|███▍      | 20800/60000 [8:06:00<14:23:45,  1.32s/step]INFO 2026-07-22 10:14:09 ot_train.py:646 step:21K smpl:1M ep:30K epch:6.10 loss:0.056 grdn:0.296 lr:7.4e-05 updt_s:0.730 data_s:0.548 smp/s:50 mem_gb:24.88
+Training:  35%|███▌      | 21000/60000 [8:10:16<13:17:13,  1.23s/step]INFO 2026-07-22 10:18:26 ot_train.py:646 step:21K smpl:1M ep:30K epch:6.15 loss:0.057 grdn:0.297 lr:7.4e-05 updt_s:0.732 data_s:0.549 smp/s:50 mem_gb:24.88
+Training:  35%|███▌      | 21200/60000 [8:14:44<15:53:47,  1.47s/step]INFO 2026-07-22 10:22:53 ot_train.py:646 step:21K smpl:1M ep:31K epch:6.21 loss:0.054 grdn:0.289 lr:7.3e-05 updt_s:0.763 data_s:0.574 smp/s:48 mem_gb:24.88
+Training:  36%|███▌      | 21400/60000 [8:19:52<15:26:18,  1.44s/step]INFO 2026-07-22 10:28:02 ot_train.py:646 step:21K smpl:1M ep:31K epch:6.27 loss:0.055 grdn:0.294 lr:7.3e-05 updt_s:0.849 data_s:0.692 smp/s:42 mem_gb:24.88
+Training:  36%|███▌      | 21600/60000 [8:25:01<16:01:00,  1.50s/step]INFO 2026-07-22 10:33:11 ot_train.py:646 step:22K smpl:1M ep:31K epch:6.33 loss:0.056 grdn:0.297 lr:7.2e-05 updt_s:0.872 data_s:0.672 smp/s:41 mem_gb:24.88
+Training:  36%|███▋      | 21800/60000 [8:30:05<16:20:32,  1.54s/step]INFO 2026-07-22 10:38:14 ot_train.py:646 step:22K smpl:1M ep:31K epch:6.39 loss:0.056 grdn:0.285 lr:7.2e-05 updt_s:0.890 data_s:0.627 smp/s:42 mem_gb:24.88
+Training:  37%|███▋      | 22000/60000 [8:35:11<16:22:35,  1.55s/step]INFO 2026-07-22 10:43:20 ot_train.py:646 step:22K smpl:1M ep:32K epch:6.45 loss:0.055 grdn:0.283 lr:7.1e-05 updt_s:0.888 data_s:0.640 smp/s:42 mem_gb:24.88
+Training:  37%|███▋      | 22200/60000 [8:40:14<16:03:38,  1.53s/step]INFO 2026-07-22 10:48:24 ot_train.py:646 step:22K smpl:1M ep:32K epch:6.51 loss:0.055 grdn:0.282 lr:7.1e-05 updt_s:0.873 data_s:0.643 smp/s:42 mem_gb:24.88
+Training:  37%|███▋      | 22400/60000 [8:45:21<16:01:42,  1.53s/step]INFO 2026-07-22 10:53:30 ot_train.py:646 step:22K smpl:1M ep:32K epch:6.57 loss:0.056 grdn:0.292 lr:7.0e-05 updt_s:0.886 data_s:0.645 smp/s:42 mem_gb:24.88
+Training:  38%|███▊      | 22600/60000 [8:50:34<16:11:00,  1.56s/step]INFO 2026-07-22 10:58:44 ot_train.py:646 step:23K smpl:1M ep:33K epch:6.62 loss:0.056 grdn:0.296 lr:7.0e-05 updt_s:0.901 data_s:0.666 smp/s:41 mem_gb:24.86
+Training:  38%|███▊      | 22800/60000 [8:55:43<16:13:58,  1.57s/step]INFO 2026-07-22 11:03:53 ot_train.py:646 step:23K smpl:1M ep:33K epch:6.68 loss:0.054 grdn:0.282 lr:6.9e-05 updt_s:0.891 data_s:0.653 smp/s:41 mem_gb:24.88
+Training:  38%|███▊      | 23000/60000 [9:00:53<16:19:17,  1.59s/step]INFO 2026-07-22 11:09:02 ot_train.py:646 step:23K smpl:1M ep:33K epch:6.74 loss:0.053 grdn:0.286 lr:6.9e-05 updt_s:0.907 data_s:0.640 smp/s:41 mem_gb:24.88
+Training:  39%|███▊      | 23200/60000 [9:05:42<14:14:54,  1.39s/step]INFO 2026-07-22 11:13:52 ot_train.py:646 step:23K smpl:1M ep:34K epch:6.80 loss:0.053 grdn:0.285 lr:6.8e-05 updt_s:0.796 data_s:0.649 smp/s:44 mem_gb:24.88
+Training:  39%|███▉      | 23400/60000 [9:10:20<14:01:20,  1.38s/step]INFO 2026-07-22 11:18:29 ot_train.py:646 step:23K smpl:1M ep:34K epch:6.86 loss:0.053 grdn:0.290 lr:6.8e-05 updt_s:0.746 data_s:0.643 smp/s:46 mem_gb:24.88
+Training:  39%|███▉      | 23600/60000 [9:14:59<13:50:06,  1.37s/step]INFO 2026-07-22 11:23:09 ot_train.py:646 step:24K smpl:2M ep:34K epch:6.92 loss:0.053 grdn:0.279 lr:6.7e-05 updt_s:0.741 data_s:0.654 smp/s:46 mem_gb:24.88
+Training:  40%|██���▉      | 23800/60000 [9:19:47<14:06:22,  1.40s/step]INFO 2026-07-22 11:27:57 ot_train.py:646 step:24K smpl:2M ep:34K epch:6.98 loss:0.053 grdn:0.283 lr:6.7e-05 updt_s:0.764 data_s:0.675 smp/s:44 mem_gb:24.88
+Training:  40%|████      | 24000/60000 [9:24:37<13:43:49,  1.37s/step]INFO 2026-07-22 11:32:46 ot_train.py:646 step:24K smpl:2M ep:35K epch:7.03 loss:0.051 grdn:0.290 lr:6.6e-05 updt_s:0.766 data_s:0.679 smp/s:44 mem_gb:24.88
+Training:  40%|████      | 24200/60000 [9:29:13<14:00:11,  1.41s/step]INFO 2026-07-22 11:37:22 ot_train.py:646 step:24K smpl:2M ep:35K epch:7.09 loss:0.052 grdn:0.290 lr:6.6e-05 updt_s:0.725 data_s:0.652 smp/s:46 mem_gb:24.88
+Training:  41%|████      | 24400/60000 [9:33:56<13:54:38,  1.41s/step]INFO 2026-07-22 11:42:05 ot_train.py:646 step:24K smpl:2M ep:35K epch:7.15 loss:0.052 grdn:0.285 lr:6.5e-05 updt_s:0.747 data_s:0.667 smp/s:45 mem_gb:24.88
+Training:  41%|████      | 24600/60000 [9:38:40<14:29:37,  1.47s/step]INFO 2026-07-22 11:46:49 ot_train.py:646 step:25K smpl:2M ep:36K epch:7.21 loss:0.053 grdn:0.290 lr:6.5e-05 updt_s:0.755 data_s:0.665 smp/s:45 mem_gb:24.88
+Training:  41%|████▏     | 24800/60000 [9:43:22<14:48:20,  1.51s/step]INFO 2026-07-22 11:51:31 ot_train.py:646 step:25K smpl:2M ep:36K epch:7.27 loss:0.053 grdn:0.303 lr:6.4e-05 updt_s:0.750 data_s:0.658 smp/s:45 mem_gb:24.88
+Training:  42%|████▏     | 25000/60000 [9:48:20<15:33:02,  1.60s/step]INFO 2026-07-22 11:56:29 ot_train.py:646 step:25K smpl:2M ep:36K epch:7.33 loss:0.050 grdn:0.284 lr:6.4e-05 updt_s:0.795 data_s:0.691 smp/s:43 mem_gb:24.86
+Training:  42%|████▏     | 25200/60000 [9:53:18<13:45:52,  1.42s/step]INFO 2026-07-22 12:01:27 ot_train.py:646 step:25K smpl:2M ep:36K epch:7.39 loss:0.050 grdn:0.281 lr:6.3e-05 updt_s:0.818 data_s:0.671 smp/s:43 mem_gb:24.88
+Training:  42%|████▏     | 25400/60000 [9:58:00<13:57:37,  1.45s/step]INFO 2026-07-22 12:06:10 ot_train.py:646 step:25K smpl:2M ep:37K epch:7.44 loss:0.050 grdn:0.287 lr:6.3e-05 updt_s:0.759 data_s:0.652 smp/s:45 mem_gb:24.88
+Training:  43%|████▎     | 25600/60000 [10:02:39<13:39:59,  1.43s/step]INFO 2026-07-22 12:10:48 ot_train.py:646 step:26K smpl:2M ep:37K epch:7.50 loss:0.051 grdn:0.287 lr:6.2e-05 updt_s:0.734 data_s:0.658 smp/s:46 mem_gb:24.88
+Training:  43%|████▎     | 25800/60000 [10:07:20<15:54:11,  1.67s/step]INFO 2026-07-22 12:15:30 ot_train.py:646 step:26K smpl:2M ep:37K epch:7.56 loss:0.050 grdn:0.289 lr:6.2e-05 updt_s:0.746 data_s:0.660 smp/s:46 mem_gb:24.88
+Training:  43%|████▎     | 26000/60000 [10:12:01<12:40:47,  1.34s/step]INFO 2026-07-22 12:20:11 ot_train.py:646 step:26K smpl:2M ep:38K epch:7.62 loss:0.049 grdn:0.290 lr:6.1e-05 updt_s:0.735 data_s:0.666 smp/s:46 mem_gb:24.88
+Training:  44%|████▎     | 26200/60000 [10:16:53<14:08:28,  1.51s/step]INFO 2026-07-22 12:25:02 ot_train.py:646 step:26K smpl:2M ep:38K epch:7.68 loss:0.050 grdn:0.289 lr:6.1e-05 updt_s:0.765 data_s:0.692 smp/s:44 mem_gb:24.88
+Training:  44%|████▍     | 26400/60000 [10:21:40<13:41:57,  1.47s/step]INFO 2026-07-22 12:29:49 ot_train.py:646 step:26K smpl:2M ep:38K epch:7.74 loss:0.051 grdn:0.299 lr:6.0e-05 updt_s:0.746 data_s:0.689 smp/s:45 mem_gb:24.88
+Training:  44%|████▍     | 26600/60000 [10:26:24<13:13:58,  1.43s/step]INFO 2026-07-22 12:34:34 ot_train.py:646 step:27K smpl:2M ep:38K epch:7.80 loss:0.050 grdn:0.286 lr:6.0e-05 updt_s:0.751 data_s:0.668 smp/s:45 mem_gb:24.88
+Training:  45%|████▍     | 26800/60000 [10:31:02<12:57:31,  1.41s/step]INFO 2026-07-22 12:39:11 ot_train.py:646 step:27K smpl:2M ep:39K epch:7.85 loss:0.049 grdn:0.283 lr:5.9e-05 updt_s:0.731 data_s:0.657 smp/s:46 mem_gb:24.88
+Training:  45%|████▌     | 27000/60000 [10:35:43<12:52:32,  1.40s/step]INFO 2026-07-22 12:43:53 ot_train.py:646 step:27K smpl:2M ep:39K epch:7.91 loss:0.050 grdn:0.293 lr:5.9e-05 updt_s:0.740 data_s:0.665 smp/s:46 mem_gb:24.88
+Training:  45%|████▌     | 27200/60000 [10:40:24<12:18:21,  1.35s/step]INFO 2026-07-22 12:48:33 ot_train.py:646 step:27K smpl:2M ep:39K epch:7.97 loss:0.048 grdn:0.287 lr:5.8e-05 updt_s:0.732 data_s:0.670 smp/s:46 mem_gb:24.86
+Training:  46%|████▌     | 27400/60000 [10:45:08<12:35:56,  1.39s/step]INFO 2026-07-22 12:53:17 ot_train.py:646 step:27K smpl:2M ep:40K epch:8.03 loss:0.047 grdn:0.285 lr:5.8e-05 updt_s:0.753 data_s:0.664 smp/s:45 mem_gb:24.88
+Training:  46%|████▌     | 27600/60000 [10:49:30<11:01:36,  1.23s/step]INFO 2026-07-22 12:57:39 ot_train.py:646 step:28K smpl:2M ep:40K epch:8.09 loss:0.047 grdn:0.292 lr:5.7e-05 updt_s:0.698 data_s:0.610 smp/s:49 mem_gb:24.88
+Training:  46%|████▋     | 27800/60000 [10:53:35<10:47:52,  1.21s/step]INFO 2026-07-22 13:01:44 ot_train.py:646 step:28K smpl:2M ep:40K epch:8.15 loss:0.046 grdn:0.296 lr:5.7e-05 updt_s:0.661 data_s:0.564 smp/s:52 mem_gb:24.88
+Training:  47%|████▋     | 28000/60000 [10:57:49<10:56:27,  1.23s/step]INFO 2026-07-22 13:05:58 ot_train.py:646 step:28K smpl:2M ep:40K epch:8.21 loss:0.047 grdn:0.297 lr:5.6e-05 updt_s:0.674 data_s:0.592 smp/s:51 mem_gb:24.88
+Training:  47%|████▋     | 28200/60000 [11:01:56<10:39:32,  1.21s/step]INFO 2026-07-22 13:10:06 ot_train.py:646 step:28K smpl:2M ep:41K epch:8.26 loss:0.046 grdn:0.287 lr:5.6e-05 updt_s:0.660 data_s:0.578 smp/s:52 mem_gb:24.88
+Training:  47%|████▋     | 28400/60000 [11:06:30<14:06:15,  1.61s/step]INFO 2026-07-22 13:14:39 ot_train.py:646 step:28K smpl:2M ep:41K epch:8.32 loss:0.045 grdn:0.287 lr:5.5e-05 updt_s:0.719 data_s:0.648 smp/s:47 mem_gb:24.88
+Training:  48%|████▊     | 28600/60000 [11:11:02<11:42:32,  1.34s/step]INFO 2026-07-22 13:19:11 ot_train.py:646 step:29K smpl:2M ep:41K epch:8.38 loss:0.047 grdn:0.302 lr:5.5e-05 updt_s:0.723 data_s:0.634 smp/s:47 mem_gb:24.88
+Training:  48%|████▊     | 28800/60000 [11:15:36<12:04:26,  1.39s/step]INFO 2026-07-22 13:23:45 ot_train.py:646 step:29K smpl:2M ep:42K epch:8.44 loss:0.046 grdn:0.290 lr:5.4e-05 updt_s:0.716 data_s:0.652 smp/s:47 mem_gb:24.88
+Training:  48%|████▊     | 29000/60000 [11:20:02<11:33:37,  1.34s/step]INFO 2026-07-22 13:28:11 ot_train.py:646 step:29K smpl:2M ep:42K epch:8.50 loss:0.046 grdn:0.296 lr:5.4e-05 updt_s:0.699 data_s:0.629 smp/s:48 mem_gb:24.88
+Training:  49%|████▊     | 29200/60000 [11:24:35<11:20:18,  1.33s/step]INFO 2026-07-22 13:32:44 ot_train.py:646 step:29K smpl:2M ep:42K epch:8.56 loss:0.046 grdn:0.290 lr:5.3e-05 updt_s:0.717 data_s:0.646 smp/s:47 mem_gb:24.88
+Training:  49%|████▉     | 29400/60000 [11:29:18<11:33:05,  1.36s/step]INFO 2026-07-22 13:37:28 ot_train.py:646 step:29K smpl:2M ep:42K epch:8.62 loss:0.045 grdn:0.301 lr:5.3e-05 updt_s:0.757 data_s:0.661 smp/s:45 mem_gb:24.86
+Training:  49%|████▉     | 29600/60000 [11:34:02<12:46:10,  1.51s/step]INFO 2026-07-22 13:42:11 ot_train.py:646 step:30K smpl:2M ep:43K epch:8.68 loss:0.045 grdn:0.305 lr:5.2e-05 updt_s:0.739 data_s:0.675 smp/s:45 mem_gb:24.88
+Training:  50%|████▉     | 29800/60000 [11:38:57<11:47:19,  1.41s/step]INFO 2026-07-22 13:47:06 ot_train.py:646 step:30K smpl:2M ep:43K epch:8.73 loss:0.044 grdn:0.282 lr:5.1e-05 updt_s:0.764 data_s:0.712 smp/s:43 mem_gb:24.88
+Training:  50%|█████     | 30000/60000 [11:43:44<11:26:03,  1.37s/step]INFO 2026-07-22 13:51:53 ot_train.py:646 step:30K smpl:2M ep:43K epch:8.79 loss:0.043 grdn:0.286 lr:5.1e-05 updt_s:0.744 data_s:0.690 smp/s:45 mem_gb:24.88
+INFO 2026-07-22 13:51:53 ot_train.py:691 Checkpoint policy after step 30000
+INFO 2026-07-22 13:52:15 in_yaml.py:1270 Saved config, action-space, prompt, and phase provenance in /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline and /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000
+Training:  50%|█████     | 30200/60000 [11:48:57<11:36:46,  1.40s/step]INFO 2026-07-22 13:57:06 ot_train.py:646 step:30K smpl:2M ep:44K epch:8.85 loss:0.044 grdn:0.285 lr:5.0e-05 updt_s:0.768 data_s:0.686 smp/s:44 mem_gb:24.88
+Training:  51%|█████     | 30400/60000 [11:53:43<11:31:59,  1.40s/step]INFO 2026-07-22 14:01:52 ot_train.py:646 step:30K smpl:2M ep:44K epch:8.91 loss:0.043 grdn:0.280 lr:5.0e-05 updt_s:0.745 data_s:0.683 smp/s:45 mem_gb:24.88
+Training:  51%|█████     | 30600/60000 [11:58:29<11:47:56,  1.44s/step]INFO 2026-07-22 14:06:39 ot_train.py:646 step:31K smpl:2M ep:44K epch:8.97 loss:0.044 grdn:0.296 lr:4.9e-05 updt_s:0.756 data_s:0.675 smp/s:45 mem_gb:24.88
+Training:  51%|█████▏    | 30800/60000 [12:03:46<12:09:01,  1.50s/step]INFO 2026-07-22 14:11:55 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.03 loss:0.044 grdn:0.283 lr:4.9e-05 updt_s:0.766 data_s:0.816 smp/s:40 mem_gb:24.88
+Training:  52%|█████▏    | 31000/60000 [12:09:07<13:21:13,  1.66s/step]INFO 2026-07-22 14:17:17 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.09 loss:0.044 grdn:0.294 lr:4.8e-05 updt_s:0.741 data_s:0.865 smp/s:40 mem_gb:24.88
+Training:  52%|█████▏    | 31200/60000 [12:14:49<18:49:57,  2.35s/step]INFO 2026-07-22 14:22:58 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.14 loss:0.044 grdn:0.290 lr:4.8e-05 updt_s:0.722 data_s:0.983 smp/s:38 mem_gb:24.88
+Training:  52%|█████▏    | 31400/60000 [12:24:03<12:42:22,  1.60s/step]INFO 2026-07-22 14:32:13 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.20 loss:0.044 grdn:0.303 lr:4.7e-05 updt_s:0.788 data_s:1.984 smp/s:23 mem_gb:24.88
+Training:  53%|█████▎    | 31600/60000 [12:29:18<14:47:59,  1.88s/step]INFO 2026-07-22 14:37:28 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.26 loss:0.044 grdn:0.292 lr:4.7e-05 updt_s:0.785 data_s:0.789 smp/s:41 mem_gb:24.86
+Training:  53%|█████▎    | 31800/60000 [12:35:03<12:30:52,  1.60s/step]INFO 2026-07-22 14:43:12 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.32 loss:0.041 grdn:0.287 lr:4.6e-05 updt_s:0.788 data_s:0.931 smp/s:37 mem_gb:24.88
+Training:  53%|█████▎    | 32000/60000 [12:41:10<14:22:55,  1.85s/step]INFO 2026-07-22 14:49:19 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.38 loss:0.040 grdn:0.290 lr:4.6e-05 updt_s:0.761 data_s:1.073 smp/s:35 mem_gb:24.88
+Training:  54%|█████▎    | 32200/60000 [12:46:37<11:54:17,  1.54s/step]INFO 2026-07-22 14:54:46 ot_train.py:646 step:32K smpl:2M ep:47K epch:9.44 loss:0.041 grdn:0.295 lr:4.5e-05 updt_s:0.768 data_s:0.866 smp/s:39 mem_gb:24.88
+Training:  54%|█████▍    | 32400/60000 [12:52:10<12:02:31,  1.57s/step]INFO 2026-07-22 15:00:20 ot_train.py:646 step:32K smpl:2M ep:47K epch:9.50 loss:0.041 grdn:0.308 lr:4.5e-05 updt_s:0.769 data_s:0.897 smp/s:38 mem_gb:24.88
+Training:  54%|█████▍    | 32600/60000 [12:57:34<14:03:50,  1.85s/step]INFO 2026-07-22 15:05:44 ot_train.py:646 step:33K smpl:2M ep:47K epch:9.55 loss:0.040 grdn:0.300 lr:4.4e-05 updt_s:0.762 data_s:0.856 smp/s:40 mem_gb:24.88
+Training:  55%|█████▍    | 32800/60000 [13:03:03<11:52:48,  1.57s/step]INFO 2026-07-22 15:11:12 ot_train.py:646 step:33K smpl:2M ep:47K epch:9.61 loss:0.040 grdn:0.303 lr:4.4e-05 updt_s:0.770 data_s:0.872 smp/s:39 mem_gb:24.88
+Training:  55%|█████▌    | 33000/60000 [13:08:38<12:59:53,  1.73s/step]INFO 2026-07-22 15:16:48 ot_train.py:646 step:33K smpl:2M ep:48K epch:9.67 loss:0.042 grdn:0.295 lr:4.3e-05 updt_s:0.743 data_s:0.931 smp/s:38 mem_gb:24.88
+Training:  55%|█████▌    | 33200/60000 [13:14:16<11:33:29,  1.55s/step]INFO 2026-07-22 15:22:25 ot_train.py:646 step:33K smpl:2M ep:48K epch:9.73 loss:0.041 grdn:0.294 lr:4.3e-05 updt_s:0.725 data_s:0.961 smp/s:38 mem_gb:24.88
+Training:  56%|█████▌    | 33400/60000 [13:19:56<11:32:29,  1.56s/step]INFO 2026-07-22 15:28:05 ot_train.py:646 step:33K smpl:2M ep:48K epch:9.79 loss:0.040 grdn:0.294 lr:4.2e-05 updt_s:0.728 data_s:0.972 smp/s:38 mem_gb:24.88
+Training:  56%|█████▌    | 33600/60000 [13:25:08<12:02:53,  1.64s/step]INFO 2026-07-22 15:33:18 ot_train.py:646 step:34K smpl:2M ep:49K epch:9.85 loss:0.040 grdn:0.294 lr:4.1e-05 updt_s:0.743 data_s:0.817 smp/s:41 mem_gb:24.88
+Training:  56%|█████▋    | 33800/60000 [13:30:26<10:54:32,  1.50s/step]INFO 2026-07-22 15:38:35 ot_train.py:646 step:34K smpl:2M ep:49K epch:9.91 loss:0.040 grdn:0.297 lr:4.1e-05 updt_s:0.773 data_s:0.812 smp/s:40 mem_gb:24.88
+Training:  57%|█████▋    | 34000/60000 [13:36:00<15:31:44,  2.15s/step]INFO 2026-07-22 15:44:09 ot_train.py:646 step:34K smpl:2M ep:49K epch:9.96 loss:0.040 grdn:0.298 lr:4.0e-05 updt_s:0.754 data_s:0.913 smp/s:38 mem_gb:24.86
+Training:  57%|█████▋    | 34200/60000 [13:41:35<10:47:58,  1.51s/step]INFO 2026-07-22 15:49:44 ot_train.py:646 step:34K smpl:2M ep:49K epch:10.02 loss:0.038 grdn:0.294 lr:4.0e-05 updt_s:0.727 data_s:0.947 smp/s:38 mem_gb:24.88
+Training:  57%|█████▋    | 34400/60000 [13:46:39<12:07:09,  1.70s/step]INFO 2026-07-22 15:54:48 ot_train.py:646 step:34K smpl:2M ep:50K epch:10.08 loss:0.037 grdn:0.298 lr:3.9e-05 updt_s:0.741 data_s:0.777 smp/s:42 mem_gb:24.88
+Training:  58%|█████▊    | 34600/60000 [13:51:56<10:08:27,  1.44s/step]INFO 2026-07-22 16:00:06 ot_train.py:646 step:35K smpl:2M ep:50K epch:10.14 loss:0.038 grdn:0.300 lr:3.9e-05 updt_s:0.731 data_s:0.855 smp/s:40 mem_gb:24.88
+Training:  58%|█████▊    | 34800/60000 [13:56:58<11:00:25,  1.57s/step]INFO 2026-07-22 16:05:08 ot_train.py:646 step:35K smpl:2M ep:50K epch:10.20 loss:0.038 grdn:0.302 lr:3.8e-05 updt_s:0.761 data_s:0.750 smp/s:42 mem_gb:24.88
+Training:  58%|█████▊    | 35000/60000 [14:02:10<11:37:58,  1.68s/step]INFO 2026-07-22 16:10:19 ot_train.py:646 step:35K smpl:2M ep:51K epch:10.26 loss:0.038 grdn:0.303 lr:3.8e-05 updt_s:0.763 data_s:0.791 smp/s:41 mem_gb:24.88
+Training:  59%|█████▊    | 35200/60000 [14:07:16<10:32:44,  1.53s/step]INFO 2026-07-22 16:15:25 ot_train.py:646 step:35K smpl:2M ep:51K epch:10.32 loss:0.038 grdn:0.304 lr:3.7e-05 updt_s:0.764 data_s:0.766 smp/s:42 mem_gb:24.88
+Training:  59%|█████▉    | 35400/60000 [14:12:20<8:39:25,  1.27s/step]INFO 2026-07-22 16:20:30 ot_train.py:646 step:35K smpl:2M ep:51K epch:10.38 loss:0.038 grdn:0.295 lr:3.7e-05 updt_s:0.746 data_s:0.774 smp/s:42 mem_gb:24.88
+Training:  59%|█████▉    | 35600/60000 [14:16:27<8:00:40,  1.18s/step]INFO 2026-07-22 16:24:36 ot_train.py:646 step:36K smpl:2M ep:51K epch:10.43 loss:0.036 grdn:0.297 lr:3.6e-05 updt_s:0.652 data_s:0.579 smp/s:52 mem_gb:24.88
+Training:  60%|█████▉    | 35800/60000 [14:20:52<8:17:04,  1.23s/step]INFO 2026-07-22 16:29:01 ot_train.py:646 step:36K smpl:2M ep:52K epch:10.49 loss:0.037 grdn:0.301 lr:3.6e-05 updt_s:0.669 data_s:0.656 smp/s:48 mem_gb:24.88
+Training:  60%|██████    | 36000/60000 [14:26:01<8:48:44,  1.32s/step]INFO 2026-07-22 16:34:10 ot_train.py:646 step:36K smpl:2M ep:52K epch:10.55 loss:0.036 grdn:0.296 lr:3.5e-05 updt_s:0.670 data_s:0.874 smp/s:41 mem_gb:24.88
+Training:  60%|██████    | 36200/60000 [14:30:20<8:21:08,  1.26s/step]INFO 2026-07-22 16:38:30 ot_train.py:646 step:36K smpl:2M ep:52K epch:10.61 loss:0.037 grdn:0.312 lr:3.5e-05 updt_s:0.661 data_s:0.634 smp/s:49 mem_gb:24.86
+Training:  61%|██████    | 36400/60000 [14:34:27<7:58:48,  1.22s/step]INFO 2026-07-22 16:42:37 ot_train.py:646 step:36K smpl:2M ep:53K epch:10.67 loss:0.036 grdn:0.302 lr:3.4e-05 updt_s:0.660 data_s:0.574 smp/s:52 mem_gb:24.88
+Training:  61%|██████    | 36600/60000 [14:38:34<7:51:28,  1.21s/step]INFO 2026-07-22 16:46:43 ot_train.py:646 step:37K smpl:2M ep:53K epch:10.73 loss:0.035 grdn:0.305 lr:3.4e-05 updt_s:0.658 data_s:0.573 smp/s:52 mem_gb:24.88
+Training:  61%|██████▏   | 36800/60000 [14:42:40<7:52:05,  1.22s/step]INFO 2026-07-22 16:50:49 ot_train.py:646 step:37K smpl:2M ep:53K epch:10.79 loss:0.034 grdn:0.295 lr:3.3e-05 updt_s:0.653 data_s:0.576 smp/s:52 mem_gb:24.88
+Training:  62%|██████▏   | 37000/60000 [14:46:44<7:41:43,  1.20s/step]INFO 2026-07-22 16:54:54 ot_train.py:646 step:37K smpl:2M ep:53K epch:10.84 loss:0.035 grdn:0.293 lr:3.3e-05 updt_s:0.648 data_s:0.572 smp/s:52 mem_gb:24.88
+Training:  62%|██████▏   | 37200/60000 [14:50:49<7:51:36,  1.24s/step]INFO 2026-07-22 16:58:58 ot_train.py:646 step:37K smpl:2M ep:54K epch:10.90 loss:0.035 grdn:0.288 lr:3.2e-05 updt_s:0.657 data_s:0.565 smp/s:52 mem_gb:24.88
+Training:  62%|██████▏   | 37400/60000 [14:54:55<7:25:48,  1.18s/step]INFO 2026-07-22 17:03:04 ot_train.py:646 step:37K smpl:2M ep:54K epch:10.96 loss:0.035 grdn:0.299 lr:3.2e-05 updt_s:0.658 data_s:0.570 smp/s:52 mem_gb:24.87
+Training:  63%|██████▎   | 37600/60000 [14:58:59<7:36:30,  1.22s/step]INFO 2026-07-22 17:07:09 ot_train.py:646 step:38K smpl:2M ep:54K epch:11.02 loss:0.034 grdn:0.292 lr:3.1e-05 updt_s:0.658 data_s:0.563 smp/s:52 mem_gb:24.88
+Training:  63%|██████▎   | 37800/60000 [15:03:02<7:17:53,  1.18s/step]INFO 2026-07-22 17:11:12 ot_train.py:646 step:38K smpl:2M ep:55K epch:11.08 loss:0.035 grdn:0.308 lr:3.1e-05 updt_s:0.651 data_s:0.563 smp/s:53 mem_gb:24.88
+Training:  63%|██████▎   | 38000/60000 [15:07:03<7:27:23,  1.22s/step]INFO 2026-07-22 17:15:13 ot_train.py:646 step:38K smpl:2M ep:55K epch:11.14 loss:0.036 grdn:0.300 lr:3.0e-05 updt_s:0.648 data_s:0.555 smp/s:53 mem_gb:24.88
+Training:  63%|██████▎   | 38009/60000 [15:07:15<7:40:05,  1.26s/step]

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260722_020800-21sn30p5/files/requirements.txt

@@ -0,0 +1,117 @@
+tzdata==2026.3
+nvidia-cufile-cu12==1.13.1.3
+wandb==0.27.2
+six==1.17.0
+peft==0.19.1
+nvidia-curand-cu12==10.3.9.90
+nvidia-cuda-runtime-cu12==12.8.90
+datasets==4.8.5
+attrs==26.1.0
+cloudpickle==3.1.2
+packaging==25.0
+nvidia-cublas-cu12==12.8.4.1
+cmake==4.1.3
+psutil==7.2.2
+pydantic==2.13.4
+GitPython==3.1.51
+pillow==12.2.0
+pytz==2026.2
+cuda-pathfinder==1.2.2
+nvidia-cusparse-cu12==12.5.8.93
+typing_extensions==4.15.0
+urllib3==2.7.0
+pydantic_core==2.46.4
+draccus==0.10.0
+mypy_extensions==1.1.0
+dill==0.4.1
+typer==0.26.8
+decord==0.6.0
+tqdm==4.68.4
+triton==3.6.0
+idna==3.18
+requests==2.34.2
+nvidia-cudnn-cu12==9.19.0.56
+mdurl==0.1.2
+shellingham==1.5.4
+zipp==4.1.0
+timm==1.0.28
+torch==2.11.0+cu128
+tokenizers==0.22.2
+nvidia-cusparselt-cu12==0.7.1
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cusolver-cu12==11.7.3.90
+opencv-python-headless==4.13.0.92
+httpx==0.28.1
+Jinja2==3.1.6
+einops==0.8.2
+mergedeep==1.3.4
+platformdirs==4.10.0
+nvidia-nvjitlink-cu12==12.8.93
+lerobot==0.6.0
+PyYAML==6.0.3
+accelerate==1.14.0
+anyio==4.14.2
+gymnasium==1.3.0
+pip==26.1.2
+huggingface_hub==1.23.0
+diffusers==0.35.2
+annotated-doc==0.0.4
+pyarrow==25.0.0
+python-dateutil==2.9.0.post0
+termcolor==3.3.0
+nvidia-nccl-cu12==2.28.9
+sentry-sdk==2.65.0
+torchcodec==0.11.1
+pyyaml-include==1.4.1
+multiprocess==0.70.19
+wrapt==2.2.2
+typing-inspection==0.4.2
+nvidia-cuda-nvrtc-cu12==12.8.93
+Pygments==2.20.0
+nvidia-cuda-cupti-cu12==12.8.90
+certifi==2026.6.17
+setuptools==78.1.0
+charset-normalizer==3.4.9
+propcache==0.5.2
+multidict==6.7.1
+regex==2026.7.10
+frozenlist==1.8.0
+smmap==5.0.3
+hf-xet==1.5.1
+torchvision==0.26.0+cu128
+gitdb==4.0.12
+transformers==5.5.4
+rich==15.0.0
+cuda-bindings==12.9.4
+protobuf==7.35.1
+aiosignal==1.4.0
+jsonlines==4.0.0
+yarl==1.24.2
+filelock==3.29.0
+av==15.1.0
+networkx==3.6.1
+sympy==1.14.0
+wheel==0.47.0
+nvidia-nvtx-cu12==12.8.90
+xxhash==3.8.1
+toml==0.10.2
+MarkupSafe==3.0.3
+pandas==2.3.3
+safetensors==0.8.0
+click==8.4.2
+fsspec==2026.2.0
+dm-tree==0.1.10
+absl-py==2.5.0
+Farama-Notifications==0.0.6
+mpmath==1.3.0
+importlib_metadata==9.0.0
+markdown-it-py==4.2.0
+h11==0.16.0
+typing-inspect==0.9.0
+httpcore==1.0.9
+annotated-types==0.7.0
+numpy==2.2.6
+nvidia-nvshmem-cu12==3.4.5
+aiohappyeyeballs==2.7.1
+cuda-toolkit==12.8.1
+aiohttp==3.14.1

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260722_020800-21sn30p5/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2026-07-22T02:08:00.119464938Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpnpx_8n8e/port-126141.txt","pid":126141,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-22T02:08:00.120016994Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":126141}
+{"time":"2026-07-22T02:08:00.120017139Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-126141-130041-1189763398/socket","Net":"unix"}}
+{"time":"2026-07-22T02:08:00.169736727Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_i85ijjb/port-126152.txt","pid":126152,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-22T02:08:00.170930965Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":126152}
+{"time":"2026-07-22T02:08:00.170919431Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-126152-130082-3741661681/socket","Net":"unix"}}
+{"time":"2026-07-22T02:08:00.303798114Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-22T02:08:00.310000814Z","level":"INFO","msg":"handleInformInit: received","streamId":"rd8tz18f","id":"1(@)"}
+{"time":"2026-07-22T02:08:00.355199917Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-22T02:08:00.361348251Z","level":"INFO","msg":"handleInformInit: received","streamId":"21sn30p5","id":"1(@)"}
+{"time":"2026-07-22T02:08:00.773273199Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rd8tz18f","id":"1(@)"}
+{"time":"2026-07-22T02:08:00.817155993Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"21sn30p5","id":"1(@)"}
+{"time":"2026-07-22T02:08:06.409472162Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"wnw3iz6fbp6m"}
+{"time":"2026-07-22T02:08:06.509993434Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"l0fljn6r9zin"}

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260722_020800-21sn30p5/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_setup.py:_flush():81] Configure stats pid to 126152
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260722_020800-21sn30p5/logs/debug.log
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260722_020800-21sn30p5/logs/debug-internal.log
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_init.py:init():766] calling init triggers
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline', 'job_name': 'INSIGHT_6D_DiT8_Baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-22 02:08:00,138 INFO    MainThread:126152 [wandb_init.py:init():814] starting backend
+2026-07-22 02:08:00,355 INFO    MainThread:126152 [wandb_init.py:init():829] sending inform_init request
+2026-07-22 02:08:00,818 INFO    MainThread:126152 [wandb_init.py:init():834] backend started and connected
+2026-07-22 02:08:00,820 INFO    MainThread:126152 [wandb_init.py:init():904] updated telemetry
+2026-07-22 02:08:00,824 INFO    MainThread:126152 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-22 02:08:01,373 INFO    MainThread:126152 [wandb_init.py:init():972] starting run threads in backend
+2026-07-22 02:08:01,428 INFO    MainThread:126152 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-22 02:08:01,429 INFO    MainThread:126152 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-22 02:08:01,429 INFO    MainThread:126152 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-22 02:08:01,429 INFO    MainThread:126152 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-22 02:08:01,431 INFO    MainThread:126152 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/files/output.log

@@ -0,0 +1,70 @@
+INFO 2026-07-23 00:58:51 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-23 00:58:51 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/21sn30p5
+INFO 2026-07-23 00:58:51 ot_train.py:298 Creating dataset
+INFO 2026-07-23 00:58:53 ot_train.py:332 Creating policy
+INFO 2026-07-23 00:58:53 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model
+INFO 2026-07-23 00:58:53 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 13208.10it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 695/695 [00:03<00:00, 198.66it/s] 
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                 | Status     |  |
+--------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.weight       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.bias         | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.bias     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.weight     | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+Loading weights from local directory
+INFO 2026-07-23 00:59:17 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-23 00:59:21 ot_train.py:437 Output dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline
+INFO 2026-07-23 00:59:21 ot_train.py:444 cfg.steps=60000 (60K)
+INFO 2026-07-23 00:59:21 ot_train.py:445 dataset.num_frames=218367 (218K)
+INFO 2026-07-23 00:59:21 ot_train.py:446 dataset.num_episodes=4930
+INFO 2026-07-23 00:59:21 ot_train.py:449 Effective batch size: 64 x 1 = 64
+INFO 2026-07-23 00:59:21 ot_train.py:450 num_learnable_params=808512640 (809M)
+INFO 2026-07-23 00:59:21 ot_train.py:451 num_total_params=2333587585 (2B)
+INFO 2026-07-23 00:59:21 ot_train.py:492 Resuming data order at epoch 13, sample 42176
+Training:   0%|          | 0/30000 [00:00<?, ?step/s]INFO 2026-07-23 00:59:21 ot_train.py:602 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   1%|          | 200/30000 [04:30<10:53:15,  1.32s/step]INFO 2026-07-23 01:03:51 ot_train.py:646 step:30K smpl:2M ep:44K epch:8.85 loss:0.043 grdn:0.291 lr:5.0e-05 updt_s:0.739 data_s:0.612 smp/s:47 mem_gb:24.87
+Training:   1%|▏         | 400/30000 [08:54<11:12:58,  1.36s/step]INFO 2026-07-23 01:08:16 ot_train.py:646 step:30K smpl:2M ep:44K epch:8.91 loss:0.043 grdn:0.295 lr:5.0e-05 updt_s:0.742 data_s:0.580 smp/s:48 mem_gb:24.87
+Training:   2%|▏         | 600/30000 [13:18<11:03:42,  1.35s/step]INFO 2026-07-23 01:12:39 ot_train.py:646 step:31K smpl:2M ep:44K epch:8.97 loss:0.044 grdn:0.296 lr:4.9e-05 updt_s:0.735 data_s:0.582 smp/s:49 mem_gb:24.87
+Training:   3%|▎         | 800/30000 [18:02<11:23:00,  1.40s/step]INFO 2026-07-23 01:17:23 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.03 loss:0.044 grdn:0.285 lr:4.9e-05 updt_s:0.778 data_s:0.639 smp/s:45 mem_gb:24.87
+Training:   3%|▎         | 1000/30000 [23:25<12:54:18,  1.60s/step]INFO 2026-07-23 01:22:47 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.09 loss:0.042 grdn:0.283 lr:4.8e-05 updt_s:0.816 data_s:0.800 smp/s:40 mem_gb:24.87
+Training:   4%|▍         | 1145/30000 [27:30<19:41:45,  2.46s/step]Traceback (most recent call last):
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1453, in <module>
+    main()
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main
+    train_module.main()
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 813, in main
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 612, in train
+    if sample_weighter is not None:
+    ^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 299, in __call__
+    transformed_transition = self._forward(transition)
+                             ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 316, in _forward
+    transition = processor_step(transition)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/groot/processor_groot.py", line 1850, in __call__
+    video = np.stack(cams, axis=2)  # (B, T, V, H, W, C)
+            ^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/numpy/_core/shape_base.py", line 467, in stack
+    return _nx.concatenate(expanded_arrays, axis=axis, out=out,
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/files/wandb-metadata.json

@@ -0,0 +1,50 @@
+{
+  "os": "Linux-6.8.0-55-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-23T00:58:50.451520Z",
+  "args": [
+    "--config_path=/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model/train_config.json",
+    "--resume=true"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePathLocal": "my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a329728c2c9524d79b966c4a7a82ee56a4c2abf7"
+  },
+  "root": "/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline",
+  "host": "ext_minje-box",
+  "executable": "/home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 2,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "1135434260480"
+    }
+  },
+  "memory": {
+    "total": "2164176568320"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "00spweoz26l63hzys0f5f482plpkbhoe"
+}

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":55870},"train/steps":37800,"_timestamp":1.7847402722368646e+09,"train/epochs":11.078597040761654,"train/lr":3.083421446984113e-05,"train/update_s":0.6514109047697275,"train/dataloading_s":0.5634355993702047,"train/samples_per_s":52.68155259277773,"_step":37800,"train/grad_norm":0.30793055780231954,"train/gpu_mem_gb":24.87638006925583,"train/samples":2419200,"train/episodes":54617.48341095496,"train/loss":0.03277819976210594,"_runtime":55870}

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/logs/debug-core.log

@@ -0,0 +1,62 @@
+{"time":"2026-07-23T00:58:50.442526721Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp2ior1pmp/port-454.txt","pid":454,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T00:58:50.443023801Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-454-1268-2688650548/socket","Net":"unix"}}
+{"time":"2026-07-23T00:58:50.44313088Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":454}
+{"time":"2026-07-23T00:58:50.483257536Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpubgbbv96/port-430.txt","pid":430,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T00:58:50.483763212Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":430}
+{"time":"2026-07-23T00:58:50.483763342Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-430-1294-1550743546/socket","Net":"unix"}}
+{"time":"2026-07-23T00:58:50.487589039Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9cf47msf/port-438.txt","pid":438,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T00:58:50.48889134Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":438}
+{"time":"2026-07-23T00:58:50.488890648Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-438-1300-2209329374/socket","Net":"unix"}}
+{"time":"2026-07-23T00:58:50.53986185Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdlqfoxj9/port-446.txt","pid":446,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T00:58:50.541192813Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":446}
+{"time":"2026-07-23T00:58:50.541189694Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-446-1338-2725561867/socket","Net":"unix"}}
+{"time":"2026-07-23T00:58:50.628929549Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T00:58:50.635401913Z","level":"INFO","msg":"handleInformInit: received","streamId":"pgr3eari","id":"1(@)"}
+{"time":"2026-07-23T00:58:50.667984858Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T00:58:50.673941153Z","level":"INFO","msg":"handleInformInit: received","streamId":"21sn30p5","id":"1(@)"}
+{"time":"2026-07-23T00:58:50.67404224Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T00:58:50.680727357Z","level":"INFO","msg":"handleInformInit: received","streamId":"rd8tz18f","id":"1(@)"}
+{"time":"2026-07-23T00:58:50.725811626Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T00:58:50.731237223Z","level":"INFO","msg":"handleInformInit: received","streamId":"svym8k0w","id":"1(@)"}
+{"time":"2026-07-23T00:58:51.09301534Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"pgr3eari","id":"1(@)"}
+{"time":"2026-07-23T00:58:51.120058787Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rd8tz18f","id":"1(@)"}
+{"time":"2026-07-23T00:58:51.131344474Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"21sn30p5","id":"1(@)"}
+{"time":"2026-07-23T00:58:51.185176771Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"svym8k0w","id":"1(@)"}
+{"time":"2026-07-23T00:58:56.889415247Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4rih9cljf6h7"}
+{"time":"2026-07-23T00:58:56.894112928Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"w89tjw1p8ewg"}
+{"time":"2026-07-23T00:58:56.928667081Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"2jvc25k8t0qw"}
+{"time":"2026-07-23T00:58:57.00426725Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"jtm5vnlo5mvy"}
+{"time":"2026-07-23T01:26:52.217924187Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.218131434Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.218172554Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T01:26:52.218155392Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.218211496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.218261727Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-454-1268-2688650548/socket","Net":"unix"}}
+{"time":"2026-07-23T01:26:52.226525402Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.226598688Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.226616087Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.227419595Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.226627243Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T01:26:52.227486909Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-438-1300-2209329374/socket","Net":"unix"}}
+{"time":"2026-07-23T01:26:52.239974032Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.24005161Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T01:26:52.24004295Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.240094238Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.240192908Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.240180686Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-446-1338-2725561867/socket","Net":"unix"}}
+{"time":"2026-07-23T01:26:52.365092389Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.365205336Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.365271549Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T01:26:52.365280497Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T01:26:52.365368244Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-430-1294-1550743546/socket","Net":"unix"}}
+{"time":"2026-07-23T01:26:52.365381884Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T01:26:53.365610409Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T01:26:53.365639983Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T01:26:53.365656633Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T01:26:53.378096243Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T01:26:53.378116102Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T01:26:53.37812429Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T01:26:53.728940335Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T01:26:53.728959563Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T01:26:53.728969099Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T01:27:02.095210277Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/logs/debug-internal.log

@@ -0,0 +1,185 @@
+{"time":"2026-07-23T00:58:50.67407502Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-23T00:58:50.674523567Z","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-23T00:58:51.13116946Z","level":"INFO","msg":"stream: created new stream","id":"21sn30p5"}
+{"time":"2026-07-23T00:58:51.131340942Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-23T00:58:51.131374554Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-23T00:58:51.131422604Z","level":"INFO","msg":"writer: started","stream_id":"21sn30p5"}
+{"time":"2026-07-23T00:58:51.131436535Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-23T00:58:51.827561847Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-07-23T00:59:08.297425216Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T00:59:08.297654843Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":30,"uploaded_len":2}
+{"time":"2026-07-23T00:59:37.251300004Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T00:59:37.25155055Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":4,"console_offset":30,"console_lines":11}
+{"time":"2026-07-23T01:00:00.774437347Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:00:00.774644628Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:00:17.348507832Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:00:17.34872665Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:00:18.082699427Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:00:18.318755789Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:00:19.033204319Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:00:33.319449994Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:00:33.73029968Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:00:48.319361819Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":12,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:00:48.961620273Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:01:03.319045011Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:01:54.913746436Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:01:54.913952832Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T01:01:55.711048389Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:01:55.71128712Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":8,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:01:56.772495513Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:01:56.772649169Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:01:57.690015841Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:02:10.640099044Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:02:36.316166553Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:02:36.316391302Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":26,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:02:36.577308081Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:02:40.640139406Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:02:40.998944203Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:02:55.640156821Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:03:02.304763647Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:03:10.640151405Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":32,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:03:17.839821438Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:03:25.639769825Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:03:31.067611483Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:03:40.640535564Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":36,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-07-23T01:03:42.028352221Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:03:51.78175928Z","level":"WARN","msg":"handler: ignoring partial history record","step":30200,"current":37801}
+{"time":"2026-07-23T01:03:55.639549937Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":40,"console_lines":2}
+{"time":"2026-07-23T01:04:29.041072512Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:04:29.041195899Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T01:04:36.20151358Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:04:36.201702259Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":4,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:04:36.525501449Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:04:36.526104662Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:04:36.963517774Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:04:44.195640018Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:05:18.030361841Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:05:18.030509078Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T01:05:18.482621576Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:05:18.482843557Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":4,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:05:45.521040573Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:05:45.521264933Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:05:46.34078666Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:05:46.341113512Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":4,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:06:20.967979378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:06:20.968143796Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:06:21.788861644Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:06:21.78911254Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":54,"events_lines":4,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:06:22.60665294Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:06:22.606811084Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:06:52.55059983Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:06:52.550770901Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":58,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:06:54.005857754Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:06:54.006028608Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":60,"events_lines":4,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:07:17.537314015Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:07:17.537456039Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:07:19.511331988Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:07:21.719190633Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":64,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:07:55.792428434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:07:55.792602686Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T01:08:16.271801254Z","level":"WARN","msg":"handler: ignoring partial history record","step":30400,"current":37801}
+{"time":"2026-07-23T01:08:18.122096142Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:08:18.122286759Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":66,"events_lines":6,"console_offset":41,"console_lines":1}
+{"time":"2026-07-23T01:08:43.134370157Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:08:43.134590054Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":72,"events_lines":2,"console_offset":41,"console_lines":2}
+{"time":"2026-07-23T01:09:18.570707037Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:09:18.570914275Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":74,"events_lines":4,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:09:41.888855012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:09:41.889196416Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":78,"events_lines":4,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:09:58.349681222Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:09:58.349864944Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":82,"events_lines":4,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:10:32.298562655Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:10:32.298733994Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":86,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:10:47.75234985Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:10:47.752530378Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":88,"events_lines":4,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:10:50.304345242Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:10:50.30450987Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":92,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:10:53.87076098Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:10:53.870915944Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:11:03.880283481Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:11:03.880585986Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":94,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:11:47.361493816Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:11:47.361663342Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":96,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:12:21.015656975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:12:21.015941791Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":98,"events_lines":4,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:12:23.637694617Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:12:23.637945315Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":102,"events_lines":4,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:12:26.18324697Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:12:26.183484773Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":106,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:12:39.916375063Z","level":"WARN","msg":"handler: ignoring partial history record","step":30600,"current":37801}
+{"time":"2026-07-23T01:12:49.940931312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:12:49.94106747Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":42,"console_lines":1}
+{"time":"2026-07-23T01:13:23.643258289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:13:23.643462897Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":108,"events_lines":2,"console_offset":42,"console_lines":2}
+{"time":"2026-07-23T01:14:07.777631625Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:14:07.777909415Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":110,"events_lines":6,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:14:48.055036754Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:14:48.055232649Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":116,"events_lines":6,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:14:57.255096375Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:14:57.255261696Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":122,"events_lines":4,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:15:31.605211554Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:15:31.605388571Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":126,"events_lines":2,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:16:05.215922359Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:16:05.216180035Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":128,"events_lines":4,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:16:49.69634125Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:16:49.696584322Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":132,"events_lines":4,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:17:16.154985994Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:17:16.155238543Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":136,"events_lines":6,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:17:19.873968641Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:17:19.874170612Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":142,"events_lines":4,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:17:23.592800748Z","level":"WARN","msg":"handler: ignoring partial history record","step":30800,"current":37801}
+{"time":"2026-07-23T01:17:47.395698123Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:17:47.395863178Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":43,"console_lines":1}
+{"time":"2026-07-23T01:18:41.196027148Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:18:41.196305281Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":146,"events_lines":4,"console_offset":43,"console_lines":2}
+{"time":"2026-07-23T01:18:44.967501709Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:18:44.967785704Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":150,"events_lines":8,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:19:09.096940616Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:19:09.097100102Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:19:39.774502906Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:19:39.774716901Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":158,"events_lines":4,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:20:04.896487412Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:20:04.898339285Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":162,"events_lines":4,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:20:05.387935141Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:20:05.388096529Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":166,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:20:05.80043589Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:20:05.866398254Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:20:06.254596059Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:20:20.867279447Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":168,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:20:21.313876641Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:20:35.867038496Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":170,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:20:36.380081928Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:20:50.867254226Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":172,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:21:32.123439622Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:21:32.123634428Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T01:21:32.580253054Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:21:32.580510985Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":174,"events_lines":6,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:21:52.507621458Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:21:52.507858451Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:21:52.899822317Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:21:52.90008305Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":180,"events_lines":4,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:22:15.448208736Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:22:15.448412109Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:22:47.174396816Z","level":"WARN","msg":"handler: ignoring partial history record","step":31000,"current":37801}
+{"time":"2026-07-23T01:22:54.565274455Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:22:54.56545658Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":184,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-07-23T01:22:55.114516539Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:22:55.115149232Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":186,"events_lines":6,"console_offset":44,"console_lines":2}
+{"time":"2026-07-23T01:23:46.222934062Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:23:46.223228008Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:24:17.987010297Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:24:17.987300877Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":192,"events_lines":6,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:25:14.919547107Z","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/minje227_hyu-hanyang-university/lerobot/21sn30p5/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
+{"time":"2026-07-23T01:25:33.395684774Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:25:33.39616829Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":198,"events_lines":4,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:26:03.938141908Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:26:03.938596709Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":202,"events_lines":10,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:26:04.698872028Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:26:04.699164454Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":212,"events_lines":4,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:26:05.542834304Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:26:05.54305129Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:26:06.2451229Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:26:20.065201503Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":216,"events_lines":2,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:26:20.516128671Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T01:26:35.065455634Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":218,"events_lines":2,"console_offset":45,"console_lines":1}
+{"time":"2026-07-23T01:26:53.032064162Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/logs/debug.log

@@ -0,0 +1,46 @@
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_setup.py:_flush():81] Configure stats pid to 430
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/logs/debug.log
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/logs/debug-internal.log
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_init.py:init():766] calling init triggers
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline', 'job_name': 'INSIGHT_6D_DiT8_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '21sn30p5', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000', '_wandb': {}}
+2026-07-23 00:58:50,452 INFO    MainThread:430 [wandb_init.py:init():814] starting backend
+2026-07-23 00:58:50,668 INFO    MainThread:430 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 00:58:51,132 INFO    MainThread:430 [wandb_init.py:init():834] backend started and connected
+2026-07-23 00:58:51,134 INFO    MainThread:430 [wandb_init.py:init():904] updated telemetry
+2026-07-23 00:58:51,138 INFO    MainThread:430 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 00:58:51,824 INFO    MainThread:430 [wandb_init.py:init():967] run resumed
+2026-07-23 00:58:51,826 INFO    MainThread:430 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 00:58:51,878 INFO    MainThread:430 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 00:58:51,878 INFO    MainThread:430 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 00:58:51,879 INFO    MainThread:430 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 00:58:51,879 INFO    MainThread:430 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 00:58:51,881 INFO    MainThread:430 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-23 01:26:52,365 INFO    wandb-AsyncioManager-main:430 [service_client.py:_forward_responses():122] Reached EOF.
+2026-07-23 01:26:52,365 INFO    wandb-AsyncioManager-main:430 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
+2026-07-23 01:26:52,365 ERROR   wandb-AsyncioManager-main:430 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
+    await asyncio_compat.race(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
+    async with open_task_group(race=True) as tg:
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
+    await anext(self.gen)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
+    await task_group._wait_all(race=race, timeout=exit_timeout)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
+    raise exc
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
+    result = await handle.wait_async(timeout=None)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
+    response = await self._handle.wait_async(timeout=timeout)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
+    raise HandleAbandonedError()
+wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_013342-21sn30p5/files/output.log

@@ -0,0 +1,41 @@
+INFO 2026-07-23 01:33:43 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-23 01:33:43 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/21sn30p5
+INFO 2026-07-23 01:33:43 ot_train.py:298 Creating dataset
+INFO 2026-07-23 01:33:45 ot_train.py:332 Creating policy
+INFO 2026-07-23 01:33:45 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model
+INFO 2026-07-23 01:33:45 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 419.61it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 695/695 [00:01<00:00, 425.50it/s]
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                 | Status     |  |
+--------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.bias         | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.bias     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.weight       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.weight     | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+Loading weights from local directory
+INFO 2026-07-23 01:34:05 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-23 01:34:07 ot_train.py:437 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline
+INFO 2026-07-23 01:34:07 ot_train.py:444 cfg.steps=60000 (60K)
+INFO 2026-07-23 01:34:07 ot_train.py:445 dataset.num_frames=218367 (218K)
+INFO 2026-07-23 01:34:07 ot_train.py:446 dataset.num_episodes=4930
+INFO 2026-07-23 01:34:07 ot_train.py:449 Effective batch size: 64 x 1 = 64
+INFO 2026-07-23 01:34:07 ot_train.py:450 num_learnable_params=808512640 (809M)
+INFO 2026-07-23 01:34:07 ot_train.py:451 num_total_params=2333587585 (2B)
+INFO 2026-07-23 01:34:07 ot_train.py:492 Resuming data order at epoch 13, sample 42176
+Training:   0%|          | 0/30000 [00:00<?, ?step/s]INFO 2026-07-23 01:34:08 ot_train.py:602 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   0%|          | 46/30000 [01:57<19:41:45,  2.37s/step]

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_013342-21sn30p5/files/requirements.txt

@@ -0,0 +1,117 @@
+tzdata==2026.3
+nvidia-cufile-cu12==1.13.1.3
+wandb==0.27.2
+six==1.17.0
+peft==0.19.1
+nvidia-curand-cu12==10.3.9.90
+nvidia-cuda-runtime-cu12==12.8.90
+datasets==4.8.5
+attrs==26.1.0
+cloudpickle==3.1.2
+packaging==25.0
+nvidia-cublas-cu12==12.8.4.1
+cmake==4.1.3
+psutil==7.2.2
+pydantic==2.13.4
+GitPython==3.1.51
+pillow==12.2.0
+pytz==2026.2
+cuda-pathfinder==1.2.2
+nvidia-cusparse-cu12==12.5.8.93
+typing_extensions==4.15.0
+urllib3==2.7.0
+pydantic_core==2.46.4
+draccus==0.10.0
+mypy_extensions==1.1.0
+dill==0.4.1
+typer==0.26.8
+decord==0.6.0
+tqdm==4.68.4
+triton==3.6.0
+idna==3.18
+requests==2.34.2
+nvidia-cudnn-cu12==9.19.0.56
+mdurl==0.1.2
+shellingham==1.5.4
+zipp==4.1.0
+timm==1.0.28
+torch==2.11.0+cu128
+tokenizers==0.22.2
+nvidia-cusparselt-cu12==0.7.1
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cusolver-cu12==11.7.3.90
+opencv-python-headless==4.13.0.92
+httpx==0.28.1
+Jinja2==3.1.6
+einops==0.8.2
+mergedeep==1.3.4
+platformdirs==4.10.0
+nvidia-nvjitlink-cu12==12.8.93
+lerobot==0.6.0
+PyYAML==6.0.3
+accelerate==1.14.0
+anyio==4.14.2
+gymnasium==1.3.0
+pip==26.1.2
+huggingface_hub==1.23.0
+diffusers==0.35.2
+annotated-doc==0.0.4
+pyarrow==25.0.0
+python-dateutil==2.9.0.post0
+termcolor==3.3.0
+nvidia-nccl-cu12==2.28.9
+sentry-sdk==2.65.0
+torchcodec==0.11.1
+pyyaml-include==1.4.1
+multiprocess==0.70.19
+wrapt==2.2.2
+typing-inspection==0.4.2
+nvidia-cuda-nvrtc-cu12==12.8.93
+Pygments==2.20.0
+nvidia-cuda-cupti-cu12==12.8.90
+certifi==2026.6.17
+setuptools==78.1.0
+charset-normalizer==3.4.9
+propcache==0.5.2
+multidict==6.7.1
+regex==2026.7.10
+frozenlist==1.8.0
+smmap==5.0.3
+hf-xet==1.5.1
+torchvision==0.26.0+cu128
+gitdb==4.0.12
+transformers==5.5.4
+rich==15.0.0
+cuda-bindings==12.9.4
+protobuf==7.35.1
+aiosignal==1.4.0
+jsonlines==4.0.0
+yarl==1.24.2
+filelock==3.29.0
+av==15.1.0
+networkx==3.6.1
+sympy==1.14.0
+wheel==0.47.0
+nvidia-nvtx-cu12==12.8.90
+xxhash==3.8.1
+toml==0.10.2
+MarkupSafe==3.0.3
+pandas==2.3.3
+safetensors==0.8.0
+click==8.4.2
+fsspec==2026.2.0
+dm-tree==0.1.10
+absl-py==2.5.0
+Farama-Notifications==0.0.6
+mpmath==1.3.0
+importlib_metadata==9.0.0
+markdown-it-py==4.2.0
+h11==0.16.0
+typing-inspect==0.9.0
+httpcore==1.0.9
+annotated-types==0.7.0
+numpy==2.2.6
+nvidia-nvshmem-cu12==3.4.5
+aiohappyeyeballs==2.7.1
+cuda-toolkit==12.8.1
+aiohttp==3.14.1

Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_013342-21sn30p5/logs/debug.log

@@ -0,0 +1,20 @@
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_setup.py:_flush():81] Configure stats pid to 814701
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_013342-21sn30p5/logs/debug.log
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_013342-21sn30p5/logs/debug-internal.log
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:init():766] calling init triggers
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline', 'job_name': 'INSIGHT_6D_DiT8_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '21sn30p5', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000', '_wandb': {}}
+2026-07-23 01:33:42,309 INFO    MainThread:814701 [wandb_init.py:init():814] starting backend
+2026-07-23 01:33:42,527 INFO    MainThread:814701 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 01:33:42,987 INFO    MainThread:814701 [wandb_init.py:init():834] backend started and connected
+2026-07-23 01:33:42,989 INFO    MainThread:814701 [wandb_init.py:init():904] updated telemetry
+2026-07-23 01:33:42,993 INFO    MainThread:814701 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 01:33:43,715 INFO    MainThread:814701 [wandb_init.py:init():967] run resumed
+2026-07-23 01:33:43,718 INFO    MainThread:814701 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 01:33:43,819 INFO    MainThread:814701 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 01:33:43,821 INFO    MainThread:814701 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/pretrained_model/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/pretrained_model/config.json

@@ -0,0 +1,118 @@
+{
+    "type": "groot_rkd",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                16
+            ]
+        },
+        "observation.images.wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.right_shoulder": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.guide": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                10
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "pretrained_revision": null,
+    "chunk_size": 16,
+    "n_action_steps": 16,
+    "max_state_dim": 132,
+    "max_action_dim": 132,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "IDENTITY",
+        "ACTION": "IDENTITY"
+    },
+    "base_model_path": "nvidia/GR00T-N1.7-3B",
+    "action_decode_transform": null,
+    "embodiment_tag": "new_embodiment",
+    "tune_llm": false,
+    "tune_visual": false,
+    "tune_projector": true,
+    "tune_diffusion_model": true,
+    "tune_vlln": true,
+    "tune_top_llm_layers": 0,
+    "num_inference_timesteps": null,
+    "rtc_ramp_rate": null,
+    "use_flash_attention": false,
+    "use_relative_actions": false,
+    "relative_exclude_joints": [],
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-05,
+    "warmup_steps": 3000,
+    "warmup_ratio": 0.05,
+    "use_bf16": true,
+    "model_params_fp32": true,
+    "image_size": [
+        256,
+        256
+    ],
+    "tokenizer_assets_repo": null,
+    "lora_rank": 0,
+    "lora_alpha": 16,
+    "lora_dropout": 0.1,
+    "lora_full_model": false,
+    "video_backend": "decord",
+    "balance_dataset_weights": true,
+    "balance_trajectory_weights": true,
+    "dataset_paths": null,
+    "output_dir": "./tmp/gr00t",
+    "save_steps": 1000,
+    "max_steps": 10000,
+    "batch_size": 32,
+    "dataloader_num_workers": 8,
+    "report_to": "wandb",
+    "resume": false,
+    "action_expert_num_layers": 8,
+    "rkd_enabled": true,
+    "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11",
+    "rkd_student_pooling": "attention",
+    "rkd_student_attention_hidden_dim": 512,
+    "rkd_student_attention_output_dim": 512,
+    "rkd_fm_loss_weight": 1.0,
+    "rkd_loss_weight": 1.0,
+    "rkd_distance_weight": 1.0,
+    "rkd_angle_weight": 2.0
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/pretrained_model/phase_schedule.json

@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "step_interval": "[start_step, until_step)",
+  "total_steps": 60000,
+  "optimizer_lifecycle": "single_continuous_optimizer",
+  "scheduler_lifecycle": "single_continuous_scheduler",
+  "phases": [
+    {
+      "index": 0,
+      "name": "rscl_rkd_cosine_decay_60K",
+      "start_step": 0,
+      "until_step": 60000,
+      "policy": {
+        "rkd_enabled": true,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_loss_weight_end": 0.0,
+        "rkd_loss_weight_schedule": "cosine",
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+      },
+      "trainable": {
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_top_llm_layers": 0,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/pretrained_model/policy_postprocessor.json

@@ -0,0 +1,23 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "groot_action_unpack_unnormalize_v2",
+      "config": {
+        "env_action_dim": 10,
+        "normalize_min_max": true,
+        "clip_normalized_action": true,
+        "libero_gripper_action": false,
+        "libero_gripper_binarize": true
+      },
+      "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/pretrained_model/policy_preprocessor.json

@@ -0,0 +1,82 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "rkd_preserve_raw_action",
+      "config": {}
+    },
+    {
+      "registry_name": "groot_n1_7_pack_inputs_v1",
+      "config": {
+        "state_horizon": 1,
+        "action_horizon": 16,
+        "valid_action_horizon": 16,
+        "video_horizon": null,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "language_key": "task",
+        "formalize_language": true,
+        "embodiment_tag": "new_embodiment",
+        "embodiment_mapping": {
+          "oxe_droid_relative_eef_relative_joint": 24,
+          "xdof_relative_eef_relative_joint": 27,
+          "xdof_relative_eef_relative_joint_subtask": 27,
+          "real_g1_relative_eef_relative_joints": 25,
+          "real_r1_pro_sharpa_relative_eef": 26,
+          "real_r1_pro_sharpa_relative_eef_human": 26,
+          "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
+          "real_r1_pro_sharpa_relative_eef_mecka": 26,
+          "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
+          "simpler_env_google": 0,
+          "simpler_env_widowx": 1,
+          "libero_sim": 2,
+          "new_embodiment": 10
+        },
+        "normalize_min_max": true,
+        "state_dropout_prob": 0.0,
+        "clip_outliers": true,
+        "use_percentiles": false,
+        "video_modality_keys": null,
+        "raw_stats": null,
+        "modality_config": null
+      },
+      "state_file": "policy_preprocessor_step_3_groot_n1_7_pack_inputs_v1.safetensors"
+    },
+    {
+      "registry_name": "groot_n1_7_vlm_encode_v1",
+      "config": {
+        "model_name": "nvidia/Cosmos-Reason2-2B",
+        "image_crop_size": [
+          230,
+          230
+        ],
+        "image_target_size": [
+          256,
+          256
+        ],
+        "shortest_image_edge": null,
+        "crop_fraction": null,
+        "use_albumentations": false,
+        "letter_box_transform": false,
+        "device": "cuda"
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/pretrained_model/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/pretrained_model/train_config.json

@@ -0,0 +1,269 @@
+{
+    "dataset": {
+        "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+        "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "groot_rkd",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    16
+                ]
+            },
+            "observation.images.wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_shoulder": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.guide": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    10
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "pretrained_revision": null,
+        "chunk_size": 16,
+        "n_action_steps": 16,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "IDENTITY",
+            "ACTION": "IDENTITY"
+        },
+        "base_model_path": "nvidia/GR00T-N1.7-3B",
+        "action_decode_transform": null,
+        "embodiment_tag": "new_embodiment",
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true,
+        "tune_top_llm_layers": 0,
+        "num_inference_timesteps": null,
+        "rtc_ramp_rate": null,
+        "use_flash_attention": false,
+        "use_relative_actions": false,
+        "relative_exclude_joints": [],
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-05,
+        "warmup_steps": 3000,
+        "warmup_ratio": 0.05,
+        "use_bf16": true,
+        "model_params_fp32": true,
+        "image_size": [
+            256,
+            256
+        ],
+        "tokenizer_assets_repo": null,
+        "lora_rank": 0,
+        "lora_alpha": 16,
+        "lora_dropout": 0.1,
+        "lora_full_model": false,
+        "video_backend": "decord",
+        "balance_dataset_weights": true,
+        "balance_trajectory_weights": true,
+        "dataset_paths": null,
+        "output_dir": "./tmp/gr00t",
+        "save_steps": 1000,
+        "max_steps": 10000,
+        "batch_size": 32,
+        "dataloader_num_workers": 8,
+        "report_to": "wandb",
+        "resume": false,
+        "action_expert_num_layers": 8,
+        "rkd_enabled": true,
+        "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11",
+        "rkd_student_pooling": "attention",
+        "rkd_student_attention_hidden_dim": 512,
+        "rkd_student_attention_output_dim": 512,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+    },
+    "reward_model": null,
+    "output_dir": "/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K",
+    "job_name": "INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60",
+    "resume": false,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 64,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 60000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-05,
+        "grad_clip_norm": 1.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 3000,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "rd8tz18f",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": null
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/training_state/optimizer_param_groups.json

@@ -0,0 +1,249 @@
+[
+    {
+        "lr": 9.632470336074009e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 9.632470336074009e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208
+        ]
+    }
+]

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000/training_state/scheduler_state.json

@@ -0,0 +1,18 @@
+{
+    "base_lrs": [
+        0.0001,
+        0.0001
+    ],
+    "last_epoch": 10000,
+    "_step_count": 10001,
+    "_is_initial": false,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        9.632470336074009e-05,
+        9.632470336074009e-05
+    ],
+    "lr_lambdas": [
+        null,
+        null
+    ]
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/policy_postprocessor.json

@@ -0,0 +1,23 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "groot_action_unpack_unnormalize_v2",
+      "config": {
+        "env_action_dim": 10,
+        "normalize_min_max": true,
+        "clip_normalized_action": true,
+        "libero_gripper_action": false,
+        "libero_gripper_binarize": true
+      },
+      "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/train_config.json

@@ -0,0 +1,269 @@
+{
+    "dataset": {
+        "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+        "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "groot_rkd",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    16
+                ]
+            },
+            "observation.images.wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_shoulder": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.guide": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    10
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "pretrained_revision": null,
+        "chunk_size": 16,
+        "n_action_steps": 16,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "IDENTITY",
+            "ACTION": "IDENTITY"
+        },
+        "base_model_path": "nvidia/GR00T-N1.7-3B",
+        "action_decode_transform": null,
+        "embodiment_tag": "new_embodiment",
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true,
+        "tune_top_llm_layers": 0,
+        "num_inference_timesteps": null,
+        "rtc_ramp_rate": null,
+        "use_flash_attention": false,
+        "use_relative_actions": false,
+        "relative_exclude_joints": [],
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-05,
+        "warmup_steps": 3000,
+        "warmup_ratio": 0.05,
+        "use_bf16": true,
+        "model_params_fp32": true,
+        "image_size": [
+            256,
+            256
+        ],
+        "tokenizer_assets_repo": null,
+        "lora_rank": 0,
+        "lora_alpha": 16,
+        "lora_dropout": 0.1,
+        "lora_full_model": false,
+        "video_backend": "decord",
+        "balance_dataset_weights": true,
+        "balance_trajectory_weights": true,
+        "dataset_paths": null,
+        "output_dir": "./tmp/gr00t",
+        "save_steps": 1000,
+        "max_steps": 10000,
+        "batch_size": 32,
+        "dataloader_num_workers": 8,
+        "report_to": "wandb",
+        "resume": false,
+        "action_expert_num_layers": 8,
+        "rkd_enabled": true,
+        "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11",
+        "rkd_student_pooling": "attention",
+        "rkd_student_attention_hidden_dim": 512,
+        "rkd_student_attention_output_dim": 512,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+    },
+    "reward_model": null,
+    "output_dir": "/home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K",
+    "job_name": "INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60",
+    "resume": false,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 64,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 60000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-05,
+        "grad_clip_norm": 1.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 3000,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "rd8tz18f",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": null
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/training_state/optimizer_param_groups.json

@@ -0,0 +1,249 @@
+[
+    {
+        "lr": 5.4128967273616625e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 5.4128967273616625e-05,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208
+        ]
+    }
+]

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/training_state/training_step.json

@@ -0,0 +1,5 @@
+{
+    "step": 30000,
+    "num_processes": 1,
+    "batch_size": 64
+}

Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260722_020800-rd8tz18f/files/output.log

@@ -0,0 +1,226 @@
+INFO 2026-07-22 02:08:01 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-22 02:08:01 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/rd8tz18f
+INFO 2026-07-22 02:08:01 ot_train.py:298 Creating dataset
+INFO 2026-07-22 02:08:03 ot_train.py:332 Creating policy
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 10752.58it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 695/695 [00:00<00:00, 819.98it/s] 
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                 | Status     |  |
+--------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.norm1.linear.bias     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.weight       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{8...31}.ff.net.2.bias         | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+INFO 2026-07-22 02:08:09 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-22 02:08:09 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K
+INFO 2026-07-22 02:08:09 ot_train.py:446 cfg.steps=60000 (60K)
+INFO 2026-07-22 02:08:09 ot_train.py:447 dataset.num_frames=218367 (218K)
+INFO 2026-07-22 02:08:09 ot_train.py:448 dataset.num_episodes=4930
+INFO 2026-07-22 02:08:09 ot_train.py:451 Effective batch size: 64 x 1 = 64
+INFO 2026-07-22 02:08:09 ot_train.py:452 num_learnable_params=810087553 (810M)
+INFO 2026-07-22 02:08:09 ot_train.py:453 num_total_params=2333706401 (2B)
+Training:   0%|          | 0/60000 [00:00<?, ?step/s]INFO 2026-07-22 02:08:09 ot_train.py:604 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   0%|          | 200/60000 [04:17<21:52:19,  1.32s/step]INFO 2026-07-22 02:12:26 ot_train.py:649 step:200 smpl:13K ep:289 epch:0.06 loss:1.332 grdn:0.807 lr:3.3e-06 updt_s:0.708 data_s:0.576 smp/s:50 mem_gb:24.94
+Training:   1%|          | 400/60000 [08:27<20:41:10,  1.25s/step]INFO 2026-07-22 02:16:37 ot_train.py:649 step:400 smpl:26K ep:578 epch:0.12 loss:0.780 grdn:2.111 lr:1.0e-05 updt_s:0.678 data_s:0.572 smp/s:51 mem_gb:24.95
+Training:   1%|          | 600/60000 [12:35<20:38:48,  1.25s/step]INFO 2026-07-22 02:20:45 ot_train.py:649 step:600 smpl:38K ep:867 epch:0.18 loss:0.382 grdn:2.395 lr:1.7e-05 updt_s:0.671 data_s:0.568 smp/s:52 mem_gb:24.95
+Training:   1%|▏         | 800/60000 [16:44<20:18:11,  1.23s/step]INFO 2026-07-22 02:24:54 ot_train.py:649 step:800 smpl:51K ep:1K epch:0.23 loss:0.299 grdn:2.202 lr:2.3e-05 updt_s:0.673 data_s:0.569 smp/s:52 mem_gb:24.95
+Training:   2%|▏         | 1000/60000 [20:54<20:24:18,  1.25s/step]INFO 2026-07-22 02:29:03 ot_train.py:649 step:1K smpl:64K ep:1K epch:0.29 loss:0.247 grdn:1.973 lr:3.0e-05 updt_s:0.679 data_s:0.564 smp/s:51 mem_gb:24.95
+Training:   2%|▏         | 1200/60000 [25:02<20:04:30,  1.23s/step]INFO 2026-07-22 02:33:11 ot_train.py:649 step:1K smpl:77K ep:2K epch:0.35 loss:0.217 grdn:1.622 lr:3.7e-05 updt_s:0.673 data_s:0.565 smp/s:52 mem_gb:24.95
+Training:   2%|▏         | 1400/60000 [29:11<19:47:37,  1.22s/step]INFO 2026-07-22 02:37:20 ot_train.py:649 step:1K smpl:90K ep:2K epch:0.41 loss:0.201 grdn:1.374 lr:4.3e-05 updt_s:0.676 data_s:0.565 smp/s:52 mem_gb:24.95
+Training:   3%|▎         | 1600/60000 [33:19<19:51:53,  1.22s/step]INFO 2026-07-22 02:41:28 ot_train.py:649 step:2K smpl:102K ep:2K epch:0.47 loss:0.189 grdn:1.126 lr:5.0e-05 updt_s:0.675 data_s:0.562 smp/s:52 mem_gb:24.95
+Training:   3%|▎         | 1800/60000 [37:28<20:34:16,  1.27s/step]INFO 2026-07-22 02:45:37 ot_train.py:649 step:2K smpl:115K ep:3K epch:0.53 loss:0.176 grdn:0.993 lr:5.7e-05 updt_s:0.677 data_s:0.566 smp/s:51 mem_gb:24.95
+Training:   3%|▎         | 2000/60000 [41:45<20:35:14,  1.28s/step]INFO 2026-07-22 02:49:54 ot_train.py:649 step:2K smpl:128K ep:3K epch:0.59 loss:0.163 grdn:0.867 lr:6.3e-05 updt_s:0.691 data_s:0.591 smp/s:50 mem_gb:24.95
+Training:   4%|▎         | 2200/60000 [46:06<20:26:04,  1.27s/step]INFO 2026-07-22 02:54:16 ot_train.py:649 step:2K smpl:141K ep:3K epch:0.64 loss:0.158 grdn:0.802 lr:7.0e-05 updt_s:0.707 data_s:0.598 smp/s:49 mem_gb:24.95
+Training:   4%|▍         | 2400/60000 [50:23<19:47:57,  1.24s/step]INFO 2026-07-22 02:58:33 ot_train.py:649 step:2K smpl:154K ep:3K epch:0.70 loss:0.152 grdn:0.755 lr:7.7e-05 updt_s:0.727 data_s:0.555 smp/s:50 mem_gb:24.93
+Training:   4%|▍         | 2600/60000 [54:31<19:50:37,  1.24s/step]INFO 2026-07-22 03:02:41 ot_train.py:649 step:3K smpl:166K ep:4K epch:0.76 loss:0.148 grdn:0.692 lr:8.3e-05 updt_s:0.678 data_s:0.559 smp/s:52 mem_gb:24.95
+Training:   5%|▍         | 2800/60000 [58:49<25:39:55,  1.62s/step]INFO 2026-07-22 03:06:59 ot_train.py:649 step:3K smpl:179K ep:4K epch:0.82 loss:0.144 grdn:0.671 lr:9.0e-05 updt_s:0.719 data_s:0.569 smp/s:50 mem_gb:24.95
+Training:   5%|▌         | 3000/60000 [1:03:01<19:13:40,  1.21s/step]INFO 2026-07-22 03:11:10 ot_train.py:649 step:3K smpl:192K ep:4K epch:0.88 loss:0.139 grdn:0.623 lr:9.7e-05 updt_s:0.687 data_s:0.569 smp/s:51 mem_gb:24.95
+Training:   5%|▌         | 3200/60000 [1:07:15<20:52:08,  1.32s/step]INFO 2026-07-22 03:15:24 ot_train.py:649 step:3K smpl:205K ep:5K epch:0.94 loss:0.138 grdn:0.584 lr:1.0e-04 updt_s:0.703 data_s:0.563 smp/s:51 mem_gb:24.95
+Training:   6%|▌         | 3400/60000 [1:11:23<19:46:03,  1.26s/step]INFO 2026-07-22 03:19:33 ot_train.py:649 step:3K smpl:218K ep:5K epch:1.00 loss:0.134 grdn:0.559 lr:1.0e-04 updt_s:0.678 data_s:0.562 smp/s:52 mem_gb:24.95
+Training:   6%|▌         | 3600/60000 [1:16:32<19:29:32,  1.24s/step]INFO 2026-07-22 03:24:41 ot_train.py:649 step:4K smpl:230K ep:5K epch:1.06 loss:0.127 grdn:0.532 lr:1.0e-04 updt_s:0.943 data_s:0.597 smp/s:42 mem_gb:24.95
+Training:   6%|▋         | 3800/60000 [1:20:38<19:13:26,  1.23s/step]INFO 2026-07-22 03:28:47 ot_train.py:649 step:4K smpl:243K ep:5K epch:1.11 loss:0.122 grdn:0.503 lr:1.0e-04 updt_s:0.678 data_s:0.551 smp/s:52 mem_gb:24.95
+Training:   7%|▋         | 4000/60000 [1:25:04<19:13:53,  1.24s/step]INFO 2026-07-22 03:33:13 ot_train.py:649 step:4K smpl:256K ep:6K epch:1.17 loss:0.120 grdn:0.488 lr:1.0e-04 updt_s:0.766 data_s:0.561 smp/s:48 mem_gb:24.95
+Training:   7%|▋         | 4200/60000 [1:29:13<18:52:47,  1.22s/step]INFO 2026-07-22 03:37:23 ot_train.py:649 step:4K smpl:269K ep:6K epch:1.23 loss:0.119 grdn:0.487 lr:1.0e-04 updt_s:0.677 data_s:0.567 smp/s:51 mem_gb:24.95
+Training:   7%|▋         | 4400/60000 [1:33:22<18:56:28,  1.23s/step]INFO 2026-07-22 03:41:32 ot_train.py:649 step:4K smpl:282K ep:6K epch:1.29 loss:0.115 grdn:0.469 lr:1.0e-04 updt_s:0.677 data_s:0.565 smp/s:51 mem_gb:24.95
+Training:   8%|▊         | 4600/60000 [1:37:32<19:57:23,  1.30s/step]INFO 2026-07-22 03:45:42 ot_train.py:649 step:5K smpl:294K ep:7K epch:1.35 loss:0.116 grdn:0.468 lr:1.0e-04 updt_s:0.679 data_s:0.567 smp/s:51 mem_gb:24.93
+Training:   8%|▊         | 4800/60000 [1:41:41<19:09:37,  1.25s/step]INFO 2026-07-22 03:49:51 ot_train.py:649 step:5K smpl:307K ep:7K epch:1.41 loss:0.109 grdn:0.434 lr:1.0e-04 updt_s:0.680 data_s:0.564 smp/s:51 mem_gb:24.95
+Training:   8%|▊         | 5000/60000 [1:45:51<18:45:26,  1.23s/step]INFO 2026-07-22 03:54:01 ot_train.py:649 step:5K smpl:320K ep:7K epch:1.47 loss:0.110 grdn:0.447 lr:1.0e-04 updt_s:0.678 data_s:0.568 smp/s:51 mem_gb:24.95
+Training:   9%|▊         | 5200/60000 [1:50:01<19:14:48,  1.26s/step]INFO 2026-07-22 03:58:11 ot_train.py:649 step:5K smpl:333K ep:8K epch:1.52 loss:0.109 grdn:0.432 lr:1.0e-04 updt_s:0.681 data_s:0.567 smp/s:51 mem_gb:24.95
+Training:   9%|▉         | 5400/60000 [1:54:12<18:38:16,  1.23s/step]INFO 2026-07-22 04:02:21 ot_train.py:649 step:5K smpl:346K ep:8K epch:1.58 loss:0.110 grdn:0.446 lr:1.0e-04 updt_s:0.683 data_s:0.569 smp/s:51 mem_gb:24.95
+Training:   9%|▉         | 5600/60000 [1:58:21<18:29:01,  1.22s/step]INFO 2026-07-22 04:06:31 ot_train.py:649 step:6K smpl:358K ep:8K epch:1.64 loss:0.107 grdn:0.434 lr:1.0e-04 updt_s:0.673 data_s:0.570 smp/s:51 mem_gb:24.95
+Training:  10%|▉         | 5800/60000 [2:02:30<18:22:02,  1.22s/step]INFO 2026-07-22 04:10:40 ot_train.py:649 step:6K smpl:371K ep:8K epch:1.70 loss:0.104 grdn:0.410 lr:9.9e-05 updt_s:0.673 data_s:0.569 smp/s:52 mem_gb:24.95
+Training:  10%|█         | 6000/60000 [2:06:41<18:19:38,  1.22s/step]INFO 2026-07-22 04:14:50 ot_train.py:649 step:6K smpl:384K ep:9K epch:1.76 loss:0.104 grdn:0.410 lr:9.9e-05 updt_s:0.684 data_s:0.568 smp/s:51 mem_gb:24.95
+Training:  10%|█         | 6200/60000 [2:10:49<18:27:13,  1.23s/step]INFO 2026-07-22 04:18:58 ot_train.py:649 step:6K smpl:397K ep:9K epch:1.82 loss:0.104 grdn:0.411 lr:9.9e-05 updt_s:0.674 data_s:0.564 smp/s:52 mem_gb:24.95
+Training:  11%|█         | 6400/60000 [2:15:03<18:35:41,  1.25s/step]INFO 2026-07-22 04:23:13 ot_train.py:649 step:6K smpl:410K ep:9K epch:1.88 loss:0.100 grdn:0.399 lr:9.9e-05 updt_s:0.704 data_s:0.564 smp/s:50 mem_gb:24.95
+Training:  11%|█         | 6600/60000 [2:19:25<19:19:48,  1.30s/step]INFO 2026-07-22 04:27:34 ot_train.py:649 step:7K smpl:422K ep:10K epch:1.93 loss:0.101 grdn:0.397 lr:9.9e-05 updt_s:0.702 data_s:0.604 smp/s:49 mem_gb:24.95
+Training:  11%|█▏        | 6800/60000 [2:23:43<18:37:35,  1.26s/step]INFO 2026-07-22 04:31:53 ot_train.py:649 step:7K smpl:435K ep:10K epch:1.99 loss:0.097 grdn:0.387 lr:9.9e-05 updt_s:0.695 data_s:0.596 smp/s:50 mem_gb:24.93
+Training:  12%|█▏        | 7000/60000 [2:28:00<18:22:47,  1.25s/step]INFO 2026-07-22 04:36:09 ot_train.py:649 step:7K smpl:448K ep:10K epch:2.05 loss:0.100 grdn:0.398 lr:9.9e-05 updt_s:0.691 data_s:0.587 smp/s:50 mem_gb:24.95
+Training:  12%|█▏        | 7200/60000 [2:32:36<21:01:28,  1.43s/step]INFO 2026-07-22 04:40:45 ot_train.py:649 step:7K smpl:461K ep:10K epch:2.11 loss:0.098 grdn:0.403 lr:9.9e-05 updt_s:0.727 data_s:0.650 smp/s:46 mem_gb:24.95
+Training:  12%|█▏        | 7400/60000 [2:37:49<24:52:45,  1.70s/step]INFO 2026-07-22 04:45:58 ot_train.py:649 step:7K smpl:474K ep:11K epch:2.17 loss:0.096 grdn:0.384 lr:9.9e-05 updt_s:0.814 data_s:0.748 smp/s:41 mem_gb:24.95
+Training:  13%|█▎        | 7600/60000 [2:43:00<20:27:40,  1.41s/step]INFO 2026-07-22 04:51:10 ot_train.py:649 step:8K smpl:486K ep:11K epch:2.23 loss:0.098 grdn:0.402 lr:9.8e-05 updt_s:0.802 data_s:0.750 smp/s:41 mem_gb:24.95
+Training:  13%|█▎        | 7800/60000 [2:48:12<21:21:18,  1.47s/step]INFO 2026-07-22 04:56:22 ot_train.py:649 step:8K smpl:499K ep:11K epch:2.29 loss:0.096 grdn:0.384 lr:9.8e-05 updt_s:0.775 data_s:0.782 smp/s:41 mem_gb:24.95
+Training:  13%|█▎        | 8000/60000 [2:54:02<22:51:48,  1.58s/step]INFO 2026-07-22 05:02:12 ot_train.py:649 step:8K smpl:512K ep:12K epch:2.34 loss:0.094 grdn:0.377 lr:9.8e-05 updt_s:0.732 data_s:1.016 smp/s:37 mem_gb:24.95
+Training:  14%|█▎        | 8200/60000 [3:00:23<26:21:50,  1.83s/step]INFO 2026-07-22 05:08:32 ot_train.py:649 step:8K smpl:525K ep:12K epch:2.40 loss:0.095 grdn:0.378 lr:9.8e-05 updt_s:0.770 data_s:1.128 smp/s:34 mem_gb:24.95
+Training:  14%|█▍        | 8400/60000 [3:06:31<23:23:09,  1.63s/step]INFO 2026-07-22 05:14:40 ot_train.py:649 step:8K smpl:538K ep:12K epch:2.46 loss:0.093 grdn:0.372 lr:9.8e-05 updt_s:0.782 data_s:1.056 smp/s:35 mem_gb:24.95
+Training:  14%|█▍        | 8600/60000 [3:12:47<29:41:52,  2.08s/step]INFO 2026-07-22 05:20:57 ot_train.py:649 step:9K smpl:550K ep:12K epch:2.52 loss:0.094 grdn:0.393 lr:9.8e-05 updt_s:0.774 data_s:1.105 smp/s:34 mem_gb:24.95
+Training:  15%|█▍        | 8800/60000 [3:17:24<17:59:12,  1.26s/step]INFO 2026-07-22 05:25:34 ot_train.py:649 step:9K smpl:563K ep:13K epch:2.58 loss:0.094 grdn:0.374 lr:9.8e-05 updt_s:0.696 data_s:0.684 smp/s:46 mem_gb:24.95
+Training:  15%|█▌        | 9000/60000 [3:21:36<20:07:38,  1.42s/step]INFO 2026-07-22 05:29:46 ot_train.py:649 step:9K smpl:576K ep:13K epch:2.64 loss:0.090 grdn:0.362 lr:9.7e-05 updt_s:0.710 data_s:0.547 smp/s:51 mem_gb:24.95
+Training:  15%|█▌        | 9200/60000 [3:26:40<21:29:33,  1.52s/step]INFO 2026-07-22 05:34:49 ot_train.py:649 step:9K smpl:589K ep:13K epch:2.70 loss:0.089 grdn:0.359 lr:9.7e-05 updt_s:0.774 data_s:0.742 smp/s:42 mem_gb:24.93
+Training:  16%|█▌        | 9400/60000 [3:31:46<21:38:04,  1.54s/step]INFO 2026-07-22 05:39:55 ot_train.py:649 step:9K smpl:602K ep:14K epch:2.75 loss:0.088 grdn:0.362 lr:9.7e-05 updt_s:0.768 data_s:0.758 smp/s:42 mem_gb:24.95
+Training:  16%|█▌        | 9600/60000 [3:36:52<22:04:05,  1.58s/step]INFO 2026-07-22 05:45:01 ot_train.py:649 step:10K smpl:614K ep:14K epch:2.81 loss:0.089 grdn:0.365 lr:9.7e-05 updt_s:0.766 data_s:0.760 smp/s:42 mem_gb:24.95
+Training:  16%|█▋        | 9800/60000 [3:42:38<23:40:20,  1.70s/step]INFO 2026-07-22 05:50:47 ot_train.py:649 step:10K smpl:627K ep:14K epch:2.87 loss:0.090 grdn:0.371 lr:9.7e-05 updt_s:0.772 data_s:0.956 smp/s:37 mem_gb:24.95
+Training:  17%|█▋        | 10000/60000 [3:47:59<21:54:35,  1.58s/step]INFO 2026-07-22 05:56:09 ot_train.py:649 step:10K smpl:640K ep:14K epch:2.93 loss:0.088 grdn:0.359 lr:9.6e-05 updt_s:0.771 data_s:0.834 smp/s:40 mem_gb:24.95
+INFO 2026-07-22 05:56:09 ot_train.py:694 Checkpoint policy after step 10000
+INFO 2026-07-22 05:56:40 in_yaml.py:1270 Saved config, action-space, prompt, and phase provenance in /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K and /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/010000
+Training:  17%|█▋        | 10200/60000 [3:55:17<20:43:28,  1.50s/step]INFO 2026-07-22 06:03:27 ot_train.py:649 step:10K smpl:653K ep:15K epch:2.99 loss:0.087 grdn:0.366 lr:9.6e-05 updt_s:0.759 data_s:1.275 smp/s:31 mem_gb:24.95
+Training:  17%|█▋        | 10400/60000 [4:01:04<25:42:44,  1.87s/step]INFO 2026-07-22 06:09:13 ot_train.py:649 step:10K smpl:666K ep:15K epch:3.05 loss:0.085 grdn:0.360 lr:9.6e-05 updt_s:0.779 data_s:0.948 smp/s:37 mem_gb:24.95
+Training:  18%|█▊        | 10600/60000 [4:06:31<20:51:22,  1.52s/step]INFO 2026-07-22 06:14:41 ot_train.py:649 step:11K smpl:678K ep:15K epch:3.11 loss:0.086 grdn:0.353 lr:9.6e-05 updt_s:0.789 data_s:0.847 smp/s:39 mem_gb:24.95
+Training:  18%|█▊        | 10800/60000 [4:11:49<21:44:09,  1.59s/step]INFO 2026-07-22 06:19:58 ot_train.py:649 step:11K smpl:691K ep:16K epch:3.17 loss:0.083 grdn:0.357 lr:9.6e-05 updt_s:0.818 data_s:0.764 smp/s:40 mem_gb:24.95
+Training:  18%|█▊        | 11000/60000 [4:17:08<21:31:46,  1.58s/step]INFO 2026-07-22 06:25:18 ot_train.py:649 step:11K smpl:704K ep:16K epch:3.22 loss:0.082 grdn:0.359 lr:9.5e-05 updt_s:0.807 data_s:0.789 smp/s:40 mem_gb:24.95
+Training:  19%|█▊        | 11200/60000 [4:23:04<23:55:50,  1.77s/step]INFO 2026-07-22 06:31:14 ot_train.py:649 step:11K smpl:717K ep:16K epch:3.28 loss:0.085 grdn:0.356 lr:9.5e-05 updt_s:0.815 data_s:0.961 smp/s:36 mem_gb:24.95
+Training:  19%|█▉        | 11400/60000 [4:28:53<31:03:55,  2.30s/step]INFO 2026-07-22 06:37:02 ot_train.py:649 step:11K smpl:730K ep:16K epch:3.34 loss:0.083 grdn:0.351 lr:9.5e-05 updt_s:0.779 data_s:0.960 smp/s:37 mem_gb:24.93
+Training:  19%|█▉        | 11600/60000 [4:35:03<27:23:43,  2.04s/step]INFO 2026-07-22 06:43:13 ot_train.py:649 step:12K smpl:742K ep:17K epch:3.40 loss:0.081 grdn:0.359 lr:9.5e-05 updt_s:0.873 data_s:0.975 smp/s:35 mem_gb:24.95
+Training:  20%|█▉        | 11800/60000 [4:40:38<23:06:30,  1.73s/step]INFO 2026-07-22 06:48:48 ot_train.py:649 step:12K smpl:755K ep:17K epch:3.46 loss:0.082 grdn:0.350 lr:9.4e-05 updt_s:0.837 data_s:0.835 smp/s:38 mem_gb:24.95
+Training:  20%|██        | 12000/60000 [4:46:10<20:35:18,  1.54s/step]INFO 2026-07-22 06:54:19 ot_train.py:649 step:12K smpl:768K ep:17K epch:3.52 loss:0.080 grdn:0.348 lr:9.4e-05 updt_s:0.811 data_s:0.843 smp/s:39 mem_gb:24.95
+Training:  20%|██        | 12200/60000 [4:51:30<19:53:32,  1.50s/step]INFO 2026-07-22 06:59:40 ot_train.py:649 step:12K smpl:781K ep:18K epch:3.58 loss:0.079 grdn:0.347 lr:9.4e-05 updt_s:0.823 data_s:0.775 smp/s:40 mem_gb:24.95
+Training:  21%|██        | 12400/60000 [4:57:39<27:17:04,  2.06s/step]INFO 2026-07-22 07:05:48 ot_train.py:649 step:12K smpl:794K ep:18K epch:3.63 loss:0.078 grdn:0.355 lr:9.4e-05 updt_s:0.793 data_s:1.046 smp/s:35 mem_gb:24.95
+Training:  21%|██        | 12600/60000 [5:04:05<21:04:48,  1.60s/step]INFO 2026-07-22 07:12:15 ot_train.py:649 step:13K smpl:806K ep:18K epch:3.69 loss:0.080 grdn:0.346 lr:9.3e-05 updt_s:0.818 data_s:1.111 smp/s:33 mem_gb:24.95
+Training:  21%|██▏       | 12800/60000 [5:09:15<16:59:54,  1.30s/step]INFO 2026-07-22 07:17:24 ot_train.py:649 step:13K smpl:819K ep:18K epch:3.75 loss:0.080 grdn:0.359 lr:9.3e-05 updt_s:0.794 data_s:0.750 smp/s:41 mem_gb:24.95
+Training:  22%|██▏       | 13000/60000 [5:13:43<17:26:31,  1.34s/step]INFO 2026-07-22 07:21:53 ot_train.py:649 step:13K smpl:832K ep:19K epch:3.81 loss:0.079 grdn:0.349 lr:9.3e-05 updt_s:0.717 data_s:0.622 smp/s:48 mem_gb:24.95
+Training:  22%|██▏       | 13200/60000 [5:18:23<20:45:55,  1.60s/step]INFO 2026-07-22 07:26:33 ot_train.py:649 step:13K smpl:845K ep:19K epch:3.87 loss:0.079 grdn:0.344 lr:9.2e-05 updt_s:0.751 data_s:0.646 smp/s:46 mem_gb:24.95
+Training:  22%|██▏       | 13400/60000 [5:23:17<20:27:40,  1.58s/step]INFO 2026-07-22 07:31:27 ot_train.py:649 step:13K smpl:858K ep:19K epch:3.93 loss:0.079 grdn:0.344 lr:9.2e-05 updt_s:0.760 data_s:0.708 smp/s:44 mem_gb:24.95
+Training:  23%|██▎       | 13600/60000 [5:27:48<18:23:46,  1.43s/step]INFO 2026-07-22 07:35:57 ot_train.py:649 step:14K smpl:870K ep:20K epch:3.99 loss:0.077 grdn:0.340 lr:9.2e-05 updt_s:0.712 data_s:0.637 smp/s:47 mem_gb:24.93
+Training:  23%|██▎       | 13800/60000 [5:32:06<16:02:02,  1.25s/step]INFO 2026-07-22 07:40:15 ot_train.py:649 step:14K smpl:883K ep:20K epch:4.04 loss:0.073 grdn:0.334 lr:9.2e-05 updt_s:0.695 data_s:0.592 smp/s:50 mem_gb:24.95
+Training:  23%|██▎       | 14000/60000 [5:36:28<16:08:52,  1.26s/step]INFO 2026-07-22 07:44:38 ot_train.py:649 step:14K smpl:896K ep:20K epch:4.10 loss:0.075 grdn:0.354 lr:9.1e-05 updt_s:0.730 data_s:0.580 smp/s:49 mem_gb:24.95
+Training:  24%|██▎       | 14200/60000 [5:40:49<15:40:17,  1.23s/step]INFO 2026-07-22 07:48:59 ot_train.py:649 step:14K smpl:909K ep:21K epch:4.16 loss:0.071 grdn:0.328 lr:9.1e-05 updt_s:0.709 data_s:0.592 smp/s:49 mem_gb:24.95
+Training:  24%|██▍       | 14400/60000 [5:45:05<16:17:10,  1.29s/step]INFO 2026-07-22 07:53:15 ot_train.py:649 step:14K smpl:922K ep:21K epch:4.22 loss:0.074 grdn:0.335 lr:9.1e-05 updt_s:0.693 data_s:0.585 smp/s:50 mem_gb:24.95
+Training:  24%|██▍       | 14600/60000 [5:49:20<19:51:38,  1.57s/step]INFO 2026-07-22 07:57:30 ot_train.py:649 step:15K smpl:934K ep:21K epch:4.28 loss:0.074 grdn:0.345 lr:9.0e-05 updt_s:0.687 data_s:0.586 smp/s:50 mem_gb:24.95
+Training:  25%|██▍       | 14800/60000 [5:53:38<16:30:54,  1.32s/step]INFO 2026-07-22 08:01:47 ot_train.py:649 step:15K smpl:947K ep:21K epch:4.34 loss:0.076 grdn:0.352 lr:9.0e-05 updt_s:0.695 data_s:0.589 smp/s:50 mem_gb:24.95
+Training:  25%|██▌       | 15000/60000 [5:57:55<15:43:22,  1.26s/step]INFO 2026-07-22 08:06:04 ot_train.py:649 step:15K smpl:960K ep:22K epch:4.40 loss:0.072 grdn:0.334 lr:9.0e-05 updt_s:0.694 data_s:0.587 smp/s:50 mem_gb:24.95
+Training:  25%|██▌       | 15200/60000 [6:02:14<15:24:03,  1.24s/step]INFO 2026-07-22 08:10:24 ot_train.py:649 step:15K smpl:973K ep:22K epch:4.45 loss:0.075 grdn:0.344 lr:8.9e-05 updt_s:0.694 data_s:0.600 smp/s:49 mem_gb:24.95
+Training:  26%|██▌       | 15400/60000 [6:06:36<28:27:09,  2.30s/step]INFO 2026-07-22 08:14:45 ot_train.py:649 step:15K smpl:986K ep:22K epch:4.51 loss:0.073 grdn:0.339 lr:8.9e-05 updt_s:0.687 data_s:0.619 smp/s:49 mem_gb:24.95
+Training:  26%|██▌       | 15600/60000 [6:13:19<25:43:04,  2.09s/step]INFO 2026-07-22 08:21:29 ot_train.py:649 step:16K smpl:998K ep:23K epch:4.57 loss:0.071 grdn:0.326 lr:8.9e-05 updt_s:0.796 data_s:1.218 smp/s:32 mem_gb:24.95
+Training:  26%|██▋       | 15800/60000 [6:18:40<24:37:17,  2.01s/step]INFO 2026-07-22 08:26:49 ot_train.py:649 step:16K smpl:1M ep:23K epch:4.63 loss:0.070 grdn:0.328 lr:8.8e-05 updt_s:0.737 data_s:0.861 smp/s:40 mem_gb:24.93
+Training:  27%|██▋       | 16000/60000 [6:23:52<18:10:10,  1.49s/step]INFO 2026-07-22 08:32:02 ot_train.py:649 step:16K smpl:1M ep:23K epch:4.69 loss:0.070 grdn:0.341 lr:8.8e-05 updt_s:0.722 data_s:0.839 smp/s:41 mem_gb:24.95
+Training:  27%|██▋       | 16200/60000 [6:29:12<19:27:13,  1.60s/step]INFO 2026-07-22 08:37:21 ot_train.py:649 step:16K smpl:1M ep:23K epch:4.75 loss:0.069 grdn:0.328 lr:8.8e-05 updt_s:0.725 data_s:0.869 smp/s:40 mem_gb:24.95
+Training:  27%|██▋       | 16400/60000 [6:35:23<19:14:49,  1.59s/step]INFO 2026-07-22 08:43:32 ot_train.py:649 step:16K smpl:1M ep:24K epch:4.81 loss:0.070 grdn:0.341 lr:8.7e-05 updt_s:0.768 data_s:1.085 smp/s:35 mem_gb:24.95
+Training:  28%|██▊       | 16600/60000 [6:40:55<24:42:53,  2.05s/step]INFO 2026-07-22 08:49:05 ot_train.py:649 step:17K smpl:1M ep:24K epch:4.87 loss:0.069 grdn:0.338 lr:8.7e-05 updt_s:0.748 data_s:0.912 smp/s:39 mem_gb:24.95
+Training:  28%|██▊       | 16800/60000 [6:47:32<35:55:49,  2.99s/step]INFO 2026-07-22 08:55:41 ot_train.py:649 step:17K smpl:1M ep:24K epch:4.92 loss:0.067 grdn:0.324 lr:8.6e-05 updt_s:0.771 data_s:1.207 smp/s:32 mem_gb:24.95
+Training:  28%|██▊       | 17000/60000 [6:52:24<16:35:32,  1.39s/step]INFO 2026-07-22 09:00:34 ot_train.py:649 step:17K smpl:1M ep:25K epch:4.98 loss:0.068 grdn:0.339 lr:8.6e-05 updt_s:0.698 data_s:0.762 smp/s:44 mem_gb:24.95
+Training:  29%|██▊       | 17200/60000 [6:58:09<23:17:26,  1.96s/step]INFO 2026-07-22 09:06:19 ot_train.py:649 step:17K smpl:1M ep:25K epch:5.04 loss:0.067 grdn:0.343 lr:8.6e-05 updt_s:0.772 data_s:0.951 smp/s:37 mem_gb:24.95
+Training:  29%|██▉       | 17400/60000 [7:04:07<25:18:32,  2.14s/step]INFO 2026-07-22 09:12:17 ot_train.py:649 step:17K smpl:1M ep:25K epch:5.10 loss:0.068 grdn:0.332 lr:8.5e-05 updt_s:0.775 data_s:1.010 smp/s:36 mem_gb:24.95
+Training:  29%|██▉       | 17600/60000 [7:09:12<16:39:30,  1.41s/step]INFO 2026-07-22 09:17:21 ot_train.py:649 step:18K smpl:1M ep:25K epch:5.16 loss:0.069 grdn:0.345 lr:8.5e-05 updt_s:0.700 data_s:0.820 smp/s:42 mem_gb:24.95
+Training:  30%|██▉       | 17800/60000 [7:14:05<22:05:15,  1.88s/step]INFO 2026-07-22 09:22:15 ot_train.py:649 step:18K smpl:1M ep:26K epch:5.22 loss:0.068 grdn:0.341 lr:8.4e-05 updt_s:0.689 data_s:0.776 smp/s:44 mem_gb:24.95
+Training:  30%|███       | 18000/60000 [7:20:06<15:41:03,  1.34s/step]INFO 2026-07-22 09:28:16 ot_train.py:649 step:18K smpl:1M ep:26K epch:5.28 loss:0.067 grdn:0.334 lr:8.4e-05 updt_s:0.736 data_s:1.067 smp/s:35 mem_gb:24.95
+Training:  30%|███       | 18200/60000 [7:24:57<17:11:00,  1.48s/step]INFO 2026-07-22 09:33:07 ot_train.py:649 step:18K smpl:1M ep:26K epch:5.33 loss:0.064 grdn:0.332 lr:8.4e-05 updt_s:0.717 data_s:0.734 smp/s:44 mem_gb:24.93
+Training:  31%|███       | 18400/60000 [7:29:39<17:33:33,  1.52s/step]INFO 2026-07-22 09:37:49 ot_train.py:649 step:18K smpl:1M ep:27K epch:5.39 loss:0.063 grdn:0.339 lr:8.3e-05 updt_s:0.705 data_s:0.703 smp/s:45 mem_gb:24.95
+Training:  31%|███       | 18600/60000 [7:34:13<14:19:24,  1.25s/step]INFO 2026-07-22 09:42:22 ot_train.py:649 step:19K smpl:1M ep:27K epch:5.45 loss:0.062 grdn:0.321 lr:8.3e-05 updt_s:0.679 data_s:0.685 smp/s:47 mem_gb:24.95
+Training:  31%|███▏      | 18800/60000 [7:39:18<20:42:37,  1.81s/step]INFO 2026-07-22 09:47:27 ot_train.py:649 step:19K smpl:1M ep:27K epch:5.51 loss:0.064 grdn:0.352 lr:8.2e-05 updt_s:0.711 data_s:0.810 smp/s:42 mem_gb:24.95
+Training:  32%|███▏      | 19000/60000 [7:44:04<14:09:10,  1.24s/step]INFO 2026-07-22 09:52:13 ot_train.py:649 step:19K smpl:1M ep:27K epch:5.57 loss:0.066 grdn:0.347 lr:8.2e-05 updt_s:0.665 data_s:0.764 smp/s:45 mem_gb:24.95
+Training:  32%|███▏      | 19200/60000 [7:48:31<15:09:29,  1.34s/step]INFO 2026-07-22 09:56:40 ot_train.py:649 step:19K smpl:1M ep:28K epch:5.63 loss:0.063 grdn:0.340 lr:8.2e-05 updt_s:0.656 data_s:0.676 smp/s:48 mem_gb:24.95
+Training:  32%|███▏      | 19400/60000 [7:53:48<25:18:20,  2.24s/step]INFO 2026-07-22 10:01:57 ot_train.py:649 step:19K smpl:1M ep:28K epch:5.69 loss:0.061 grdn:0.333 lr:8.1e-05 updt_s:0.705 data_s:0.878 smp/s:40 mem_gb:24.95
+Training:  33%|███▎      | 19600/60000 [7:58:34<14:01:16,  1.25s/step]INFO 2026-07-22 10:06:43 ot_train.py:649 step:20K smpl:1M ep:28K epch:5.74 loss:0.063 grdn:0.336 lr:8.1e-05 updt_s:0.692 data_s:0.736 smp/s:45 mem_gb:24.95
+Training:  33%|███▎      | 19800/60000 [8:02:55<14:58:11,  1.34s/step]INFO 2026-07-22 10:11:05 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.80 loss:0.062 grdn:0.338 lr:8.0e-05 updt_s:0.672 data_s:0.633 smp/s:49 mem_gb:24.95
+Training:  33%|███▎      | 20000/60000 [8:07:14<13:57:34,  1.26s/step]INFO 2026-07-22 10:15:23 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.86 loss:0.063 grdn:0.336 lr:8.0e-05 updt_s:0.657 data_s:0.632 smp/s:50 mem_gb:24.95
+Training:  34%|███▎      | 20200/60000 [8:11:37<16:52:51,  1.53s/step]INFO 2026-07-22 10:19:46 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.92 loss:0.062 grdn:0.336 lr:7.9e-05 updt_s:0.668 data_s:0.644 smp/s:49 mem_gb:24.95
+Training:  34%|███▍      | 20400/60000 [8:17:02<22:25:14,  2.04s/step]INFO 2026-07-22 10:25:11 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.98 loss:0.060 grdn:0.338 lr:7.9e-05 updt_s:0.830 data_s:0.790 smp/s:39 mem_gb:24.93
+Training:  34%|███▍      | 20600/60000 [8:23:11<16:36:52,  1.52s/step]INFO 2026-07-22 10:31:21 ot_train.py:649 step:21K smpl:1M ep:30K epch:6.04 loss:0.059 grdn:0.336 lr:7.8e-05 updt_s:0.945 data_s:0.899 smp/s:35 mem_gb:24.95
+Training:  35%|███▍      | 20800/60000 [8:28:24<17:48:36,  1.64s/step]INFO 2026-07-22 10:36:33 ot_train.py:649 step:21K smpl:1M ep:30K epch:6.10 loss:0.060 grdn:0.339 lr:7.8e-05 updt_s:0.802 data_s:0.756 smp/s:41 mem_gb:24.95
+Training:  35%|███▌      | 21000/60000 [8:33:41<16:52:48,  1.56s/step]INFO 2026-07-22 10:41:50 ot_train.py:649 step:21K smpl:1M ep:30K epch:6.15 loss:0.058 grdn:0.330 lr:7.8e-05 updt_s:0.816 data_s:0.764 smp/s:40 mem_gb:24.95
+Training:  35%|███▌      | 21200/60000 [8:38:56<16:39:14,  1.55s/step]INFO 2026-07-22 10:47:06 ot_train.py:649 step:21K smpl:1M ep:31K epch:6.21 loss:0.057 grdn:0.327 lr:7.7e-05 updt_s:0.816 data_s:0.758 smp/s:41 mem_gb:24.95
+Training:  36%|███▌      | 21400/60000 [8:44:13<18:56:56,  1.77s/step]INFO 2026-07-22 10:52:23 ot_train.py:649 step:21K smpl:1M ep:31K epch:6.27 loss:0.057 grdn:0.318 lr:7.7e-05 updt_s:0.820 data_s:0.761 smp/s:40 mem_gb:24.95
+Training:  36%|███▌      | 21600/60000 [8:49:33<16:29:38,  1.55s/step]INFO 2026-07-22 10:57:42 ot_train.py:649 step:22K smpl:1M ep:31K epch:6.33 loss:0.058 grdn:0.328 lr:7.6e-05 updt_s:0.821 data_s:0.771 smp/s:40 mem_gb:24.95
+Training:  36%|███▋      | 21800/60000 [8:54:53<17:50:20,  1.68s/step]INFO 2026-07-22 11:03:03 ot_train.py:649 step:22K smpl:1M ep:31K epch:6.39 loss:0.057 grdn:0.320 lr:7.6e-05 updt_s:0.829 data_s:0.771 smp/s:40 mem_gb:24.95
+Training:  37%|███▋      | 22000/60000 [9:00:10<15:56:38,  1.51s/step]INFO 2026-07-22 11:08:19 ot_train.py:649 step:22K smpl:1M ep:32K epch:6.45 loss:0.058 grdn:0.331 lr:7.5e-05 updt_s:0.816 data_s:0.762 smp/s:41 mem_gb:24.95
+Training:  37%|███▋      | 22200/60000 [9:05:08<15:59:10,  1.52s/step]INFO 2026-07-22 11:13:18 ot_train.py:649 step:22K smpl:1M ep:32K epch:6.51 loss:0.058 grdn:0.335 lr:7.5e-05 updt_s:0.797 data_s:0.691 smp/s:43 mem_gb:24.95
+Training:  37%|███▋      | 22400/60000 [9:09:46<14:22:35,  1.38s/step]INFO 2026-07-22 11:17:56 ot_train.py:649 step:22K smpl:1M ep:32K epch:6.57 loss:0.058 grdn:0.335 lr:7.4e-05 updt_s:0.744 data_s:0.642 smp/s:46 mem_gb:24.95
+Training:  38%|███▊      | 22600/60000 [9:14:26<15:09:36,  1.46s/step]INFO 2026-07-22 11:22:35 ot_train.py:649 step:23K smpl:1M ep:33K epch:6.62 loss:0.058 grdn:0.338 lr:7.4e-05 updt_s:0.746 data_s:0.649 smp/s:46 mem_gb:24.93
+Training:  38%|███▊      | 22800/60000 [9:19:16<14:36:22,  1.41s/step]INFO 2026-07-22 11:27:25 ot_train.py:649 step:23K smpl:1M ep:33K epch:6.68 loss:0.054 grdn:0.317 lr:7.3e-05 updt_s:0.781 data_s:0.666 smp/s:44 mem_gb:24.95
+Training:  38%|███▊      | 23000/60000 [9:24:06<14:43:05,  1.43s/step]INFO 2026-07-22 11:32:16 ot_train.py:649 step:23K smpl:1M ep:33K epch:6.74 loss:0.054 grdn:0.337 lr:7.3e-05 updt_s:0.771 data_s:0.676 smp/s:44 mem_gb:24.95
+Training:  39%|███▊      | 23200/60000 [9:28:41<14:08:55,  1.38s/step]INFO 2026-07-22 11:36:51 ot_train.py:649 step:23K smpl:1M ep:34K epch:6.80 loss:0.054 grdn:0.333 lr:7.2e-05 updt_s:0.731 data_s:0.643 smp/s:47 mem_gb:24.95
+Training:  39%|███▉      | 23400/60000 [9:33:24<14:19:55,  1.41s/step]INFO 2026-07-22 11:41:34 ot_train.py:649 step:23K smpl:1M ep:34K epch:6.86 loss:0.054 grdn:0.347 lr:7.2e-05 updt_s:0.748 data_s:0.662 smp/s:45 mem_gb:24.95
+Training:  39%|███▉      | 23600/60000 [9:38:08<14:07:41,  1.40s/step]INFO 2026-07-22 11:46:18 ot_train.py:649 step:24K smpl:2M ep:34K epch:6.92 loss:0.055 grdn:0.341 lr:7.1e-05 updt_s:0.758 data_s:0.660 smp/s:45 mem_gb:24.95
+Training:  40%|███▉      | 23800/60000 [9:42:50<14:15:54,  1.42s/step]INFO 2026-07-22 11:51:00 ot_train.py:649 step:24K smpl:2M ep:34K epch:6.98 loss:0.053 grdn:0.325 lr:7.1e-05 updt_s:0.749 data_s:0.658 smp/s:45 mem_gb:24.95
+Training:  40%|████      | 24000/60000 [9:47:47<14:51:34,  1.49s/step]INFO 2026-07-22 11:55:56 ot_train.py:649 step:24K smpl:2M ep:35K epch:7.03 loss:0.053 grdn:0.337 lr:7.0e-05 updt_s:0.783 data_s:0.696 smp/s:43 mem_gb:24.95
+Training:  40%|████      | 24200/60000 [9:52:49<13:53:05,  1.40s/step]INFO 2026-07-22 12:00:58 ot_train.py:649 step:24K smpl:2M ep:35K epch:7.09 loss:0.053 grdn:0.337 lr:7.0e-05 updt_s:0.785 data_s:0.721 smp/s:42 mem_gb:24.95
+Training:  41%|████      | 24400/60000 [9:57:35<13:13:12,  1.34s/step]INFO 2026-07-22 12:05:45 ot_train.py:649 step:24K smpl:2M ep:35K epch:7.15 loss:0.052 grdn:0.329 lr:6.9e-05 updt_s:0.769 data_s:0.659 smp/s:45 mem_gb:24.95
+Training:  41%|████      | 24600/60000 [10:02:14<14:07:27,  1.44s/step]INFO 2026-07-22 12:10:24 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.21 loss:0.053 grdn:0.329 lr:6.9e-05 updt_s:0.733 data_s:0.659 smp/s:46 mem_gb:24.95
+Training:  41%|████▏     | 24800/60000 [10:06:54<14:00:23,  1.43s/step]INFO 2026-07-22 12:15:04 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.27 loss:0.052 grdn:0.339 lr:6.8e-05 updt_s:0.735 data_s:0.662 smp/s:46 mem_gb:24.95
+Training:  42%|████▏     | 25000/60000 [10:11:38<13:47:27,  1.42s/step]INFO 2026-07-22 12:19:47 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.33 loss:0.051 grdn:0.332 lr:6.8e-05 updt_s:0.747 data_s:0.668 smp/s:45 mem_gb:24.93
+Training:  42%|████▏     | 25200/60000 [10:16:28<13:49:56,  1.43s/step]INFO 2026-07-22 12:24:38 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.39 loss:0.050 grdn:0.336 lr:6.7e-05 updt_s:0.764 data_s:0.684 smp/s:44 mem_gb:24.95
+Training:  42%|████▏     | 25400/60000 [10:21:16<13:52:20,  1.44s/step]INFO 2026-07-22 12:29:26 ot_train.py:649 step:25K smpl:2M ep:37K epch:7.44 loss:0.048 grdn:0.321 lr:6.7e-05 updt_s:0.754 data_s:0.685 smp/s:44 mem_gb:24.95
+Training:  43%|████▎     | 25600/60000 [10:26:01<15:14:26,  1.59s/step]INFO 2026-07-22 12:34:10 ot_train.py:649 step:26K smpl:2M ep:37K epch:7.50 loss:0.051 grdn:0.339 lr:6.6e-05 updt_s:0.754 data_s:0.664 smp/s:45 mem_gb:24.95
+Training:  43%|████▎     | 25800/60000 [10:30:40<12:58:35,  1.37s/step]INFO 2026-07-22 12:38:50 ot_train.py:649 step:26K smpl:2M ep:37K epch:7.56 loss:0.049 grdn:0.333 lr:6.6e-05 updt_s:0.730 data_s:0.664 smp/s:46 mem_gb:24.95
+Training:  43%|████▎     | 26000/60000 [10:35:21<13:05:42,  1.39s/step]INFO 2026-07-22 12:43:31 ot_train.py:649 step:26K smpl:2M ep:38K epch:7.62 loss:0.049 grdn:0.337 lr:6.5e-05 updt_s:0.733 data_s:0.668 smp/s:46 mem_gb:24.95
+Training:  44%|████▎     | 26200/60000 [10:40:03<13:10:27,  1.40s/step]INFO 2026-07-22 12:48:12 ot_train.py:649 step:26K smpl:2M ep:38K epch:7.68 loss:0.049 grdn:0.316 lr:6.5e-05 updt_s:0.747 data_s:0.657 smp/s:46 mem_gb:24.95
+Training:  44%|████▍     | 26400/60000 [10:44:46<13:42:28,  1.47s/step]INFO 2026-07-22 12:52:56 ot_train.py:649 step:26K smpl:2M ep:38K epch:7.74 loss:0.047 grdn:0.313 lr:6.4e-05 updt_s:0.746 data_s:0.668 smp/s:45 mem_gb:24.95
+Training:  44%|████▍     | 26600/60000 [10:49:11<11:23:30,  1.23s/step]INFO 2026-07-22 12:57:20 ot_train.py:649 step:27K smpl:2M ep:38K epch:7.80 loss:0.048 grdn:0.316 lr:6.4e-05 updt_s:0.705 data_s:0.615 smp/s:48 mem_gb:24.95
+Training:  45%|████▍     | 26800/60000 [10:53:16<11:14:11,  1.22s/step]INFO 2026-07-22 13:01:26 ot_train.py:649 step:27K smpl:2M ep:39K epch:7.85 loss:0.047 grdn:0.322 lr:6.3e-05 updt_s:0.662 data_s:0.562 smp/s:52 mem_gb:24.95
+Training:  45%|████▌     | 27000/60000 [10:57:29<11:07:30,  1.21s/step]INFO 2026-07-22 13:05:39 ot_train.py:649 step:27K smpl:2M ep:39K epch:7.91 loss:0.048 grdn:0.317 lr:6.3e-05 updt_s:0.686 data_s:0.577 smp/s:51 mem_gb:24.95
+Training:  45%|████▌     | 27200/60000 [11:01:39<11:23:29,  1.25s/step]INFO 2026-07-22 13:09:48 ot_train.py:649 step:27K smpl:2M ep:39K epch:7.97 loss:0.046 grdn:0.317 lr:6.2e-05 updt_s:0.664 data_s:0.581 smp/s:51 mem_gb:24.93
+Training:  46%|████▌     | 27400/60000 [11:06:09<12:24:24,  1.37s/step]INFO 2026-07-22 13:14:18 ot_train.py:649 step:27K smpl:2M ep:40K epch:8.03 loss:0.045 grdn:0.324 lr:6.1e-05 updt_s:0.716 data_s:0.631 smp/s:48 mem_gb:24.95
+Training:  46%|████▌     | 27600/60000 [11:10:43<11:57:25,  1.33s/step]INFO 2026-07-22 13:18:53 ot_train.py:649 step:28K smpl:2M ep:40K epch:8.09 loss:0.045 grdn:0.317 lr:6.1e-05 updt_s:0.731 data_s:0.641 smp/s:47 mem_gb:24.95
+Training:  46%|████▋     | 27800/60000 [11:15:17<12:01:31,  1.34s/step]INFO 2026-07-22 13:23:26 ot_train.py:649 step:28K smpl:2M ep:40K epch:8.15 loss:0.043 grdn:0.322 lr:6.0e-05 updt_s:0.717 data_s:0.645 smp/s:47 mem_gb:24.95
+Training:  47%|████▋     | 28000/60000 [11:19:43<11:21:11,  1.28s/step]INFO 2026-07-22 13:27:53 ot_train.py:649 step:28K smpl:2M ep:40K epch:8.21 loss:0.046 grdn:0.343 lr:6.0e-05 updt_s:0.697 data_s:0.633 smp/s:48 mem_gb:24.95
+Training:  47%|████▋     | 28200/60000 [11:24:16<12:02:57,  1.36s/step]INFO 2026-07-22 13:32:26 ot_train.py:649 step:28K smpl:2M ep:41K epch:8.26 loss:0.044 grdn:0.333 lr:5.9e-05 updt_s:0.716 data_s:0.648 smp/s:47 mem_gb:24.95
+Training:  47%|████▋     | 28400/60000 [11:29:00<12:03:38,  1.37s/step]INFO 2026-07-22 13:37:10 ot_train.py:649 step:28K smpl:2M ep:41K epch:8.32 loss:0.044 grdn:0.312 lr:5.9e-05 updt_s:0.755 data_s:0.659 smp/s:45 mem_gb:24.95
+Training:  48%|████▊     | 28600/60000 [11:33:42<12:17:06,  1.41s/step]INFO 2026-07-22 13:41:52 ot_train.py:649 step:29K smpl:2M ep:41K epch:8.38 loss:0.045 grdn:0.337 lr:5.8e-05 updt_s:0.728 data_s:0.679 smp/s:45 mem_gb:24.95
+Training:  48%|████▊     | 28800/60000 [11:38:37<12:49:31,  1.48s/step]INFO 2026-07-22 13:46:47 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.44 loss:0.044 grdn:0.331 lr:5.8e-05 updt_s:0.760 data_s:0.713 smp/s:43 mem_gb:24.95
+Training:  48%|████▊     | 29000/60000 [11:43:25<12:07:12,  1.41s/step]INFO 2026-07-22 13:51:35 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.50 loss:0.044 grdn:0.317 lr:5.7e-05 updt_s:0.737 data_s:0.699 smp/s:45 mem_gb:24.95
+Training:  49%|████▊     | 29200/60000 [11:48:14<12:07:07,  1.42s/step]INFO 2026-07-22 13:56:23 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.56 loss:0.044 grdn:0.308 lr:5.7e-05 updt_s:0.756 data_s:0.683 smp/s:44 mem_gb:24.95
+Training:  49%|████▉     | 29400/60000 [11:53:00<11:40:53,  1.37s/step]INFO 2026-07-22 14:01:09 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.62 loss:0.043 grdn:0.339 lr:5.6e-05 updt_s:0.743 data_s:0.685 smp/s:45 mem_gb:24.93
+Training:  49%|████▉     | 29600/60000 [11:57:47<12:15:25,  1.45s/step]INFO 2026-07-22 14:05:56 ot_train.py:649 step:30K smpl:2M ep:43K epch:8.68 loss:0.042 grdn:0.310 lr:5.5e-05 updt_s:0.754 data_s:0.677 smp/s:45 mem_gb:24.95
+Training:  50%|████▉     | 29800/60000 [12:03:05<11:55:16,  1.42s/step]INFO 2026-07-22 14:11:14 ot_train.py:649 step:30K smpl:2M ep:43K epch:8.73 loss:0.040 grdn:0.304 lr:5.5e-05 updt_s:0.765 data_s:0.820 smp/s:40 mem_gb:24.95
+Training:  50%|█████     | 30000/60000 [12:08:10<16:29:57,  1.98s/step]INFO 2026-07-22 14:16:20 ot_train.py:649 step:30K smpl:2M ep:43K epch:8.79 loss:0.040 grdn:0.327 lr:5.4e-05 updt_s:0.751 data_s:0.772 smp/s:42 mem_gb:24.95
+INFO 2026-07-22 14:16:20 ot_train.py:694 Checkpoint policy after step 30000
+INFO 2026-07-22 14:16:47 in_yaml.py:1270 Saved config, action-space, prompt, and phase provenance in /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K and /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000
+Training:  50%|█████     | 30200/60000 [12:14:20<14:38:30,  1.77s/step]INFO 2026-07-22 14:22:30 ot_train.py:649 step:30K smpl:2M ep:44K epch:8.85 loss:0.041 grdn:0.318 lr:5.4e-05 updt_s:0.723 data_s:0.986 smp/s:37 mem_gb:24.95
+Training:  51%|█████     | 30400/60000 [12:23:41<13:42:34,  1.67s/step]INFO 2026-07-22 14:31:51 ot_train.py:649 step:30K smpl:2M ep:44K epch:8.91 loss:0.039 grdn:0.292 lr:5.3e-05 updt_s:0.794 data_s:2.009 smp/s:23 mem_gb:24.95
+Training:  51%|█████     | 30600/60000 [12:28:54<12:32:43,  1.54s/step]INFO 2026-07-22 14:37:04 ot_train.py:649 step:31K smpl:2M ep:44K epch:8.97 loss:0.041 grdn:0.328 lr:5.3e-05 updt_s:0.792 data_s:0.771 smp/s:41 mem_gb:24.95
+Training:  51%|█████▏    | 30800/60000 [12:34:40<11:48:47,  1.46s/step]INFO 2026-07-22 14:42:49 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.03 loss:0.040 grdn:0.321 lr:5.2e-05 updt_s:0.799 data_s:0.923 smp/s:37 mem_gb:24.95
+Training:  52%|█████▏    | 31000/60000 [12:40:41<15:23:00,  1.91s/step]INFO 2026-07-22 14:48:51 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.09 loss:0.040 grdn:0.316 lr:5.2e-05 updt_s:0.765 data_s:1.039 smp/s:35 mem_gb:24.95
+Training:  52%|█████▏    | 31200/60000 [12:46:14<12:47:05,  1.60s/step]INFO 2026-07-22 14:54:24 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.14 loss:0.038 grdn:0.312 lr:5.1e-05 updt_s:0.765 data_s:0.897 smp/s:39 mem_gb:24.95
+Training:  52%|█████▏    | 31400/60000 [12:51:45<12:19:33,  1.55s/step]INFO 2026-07-22 14:59:55 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.20 loss:0.040 grdn:0.315 lr:5.1e-05 updt_s:0.775 data_s:0.877 smp/s:39 mem_gb:24.95
+Training:  53%|█████▎    | 31600/60000 [12:57:10<15:53:42,  2.01s/step]INFO 2026-07-22 15:05:20 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.26 loss:0.039 grdn:0.293 lr:5.0e-05 updt_s:0.767 data_s:0.854 smp/s:39 mem_gb:24.93
+Training:  53%|█████▎    | 31800/60000 [13:02:41<11:41:35,  1.49s/step]INFO 2026-07-22 15:10:50 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.32 loss:0.037 grdn:0.300 lr:4.9e-05 updt_s:0.754 data_s:0.896 smp/s:39 mem_gb:24.95
+Training:  53%|█████▎    | 32000/60000 [13:08:15<13:24:14,  1.72s/step]INFO 2026-07-22 15:16:24 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.38 loss:0.037 grdn:0.316 lr:4.9e-05 updt_s:0.735 data_s:0.930 smp/s:38 mem_gb:24.95
+Training:  54%|█████▎    | 32200/60000 [13:13:51<12:35:49,  1.63s/step]INFO 2026-07-22 15:22:01 ot_train.py:649 step:32K smpl:2M ep:47K epch:9.44 loss:0.038 grdn:0.312 lr:4.8e-05 updt_s:0.733 data_s:0.948 smp/s:38 mem_gb:24.95
+Training:  54%|█████▍    | 32400/60000 [13:19:34<12:09:59,  1.59s/step]INFO 2026-07-22 15:27:43 ot_train.py:649 step:32K smpl:2M ep:47K epch:9.50 loss:0.037 grdn:0.309 lr:4.8e-05 updt_s:0.724 data_s:0.984 smp/s:37 mem_gb:24.95
+Training:  54%|█████▍    | 32600/60000 [13:24:45<11:38:53,  1.53s/step]INFO 2026-07-22 15:32:54 ot_train.py:649 step:33K smpl:2M ep:47K epch:9.55 loss:0.037 grdn:0.300 lr:4.7e-05 updt_s:0.731 data_s:0.821 smp/s:41 mem_gb:24.95
+Training:  55%|█████▍    | 32800/60000 [13:30:01<11:39:49,  1.54s/step]INFO 2026-07-22 15:38:11 ot_train.py:649 step:33K smpl:2M ep:47K epch:9.61 loss:0.037 grdn:0.301 lr:4.7e-05 updt_s:0.765 data_s:0.814 smp/s:41 mem_gb:24.95
+Training:  55%|█████▌    | 33000/60000 [13:35:27<23:09:11,  3.09s/step]INFO 2026-07-22 15:43:36 ot_train.py:649 step:33K smpl:2M ep:48K epch:9.67 loss:0.036 grdn:0.293 lr:4.6e-05 updt_s:0.750 data_s:0.875 smp/s:39 mem_gb:24.95
+Training:  55%|█████▌    | 33200/60000 [13:41:11<11:03:16,  1.48s/step]INFO 2026-07-22 15:49:21 ot_train.py:649 step:33K smpl:2M ep:48K epch:9.73 loss:0.036 grdn:0.291 lr:4.6e-05 updt_s:0.729 data_s:0.988 smp/s:37 mem_gb:24.95
+Training:  56%|█████▌    | 33400/60000 [13:46:14<11:06:33,  1.50s/step]INFO 2026-07-22 15:54:23 ot_train.py:649 step:33K smpl:2M ep:48K epch:9.79 loss:0.036 grdn:0.301 lr:4.5e-05 updt_s:0.743 data_s:0.765 smp/s:42 mem_gb:24.95
+Training:  56%|█████▌    | 33600/60000 [13:51:33<10:32:19,  1.44s/step]INFO 2026-07-22 15:59:42 ot_train.py:649 step:34K smpl:2M ep:49K epch:9.85 loss:0.036 grdn:0.308 lr:4.4e-05 updt_s:0.732 data_s:0.862 smp/s:40 mem_gb:24.95
+Training:  56%|█████▋    | 33800/60000 [13:56:35<11:34:13,  1.59s/step]INFO 2026-07-22 16:04:44 ot_train.py:649 step:34K smpl:2M ep:49K epch:9.91 loss:0.036 grdn:0.296 lr:4.4e-05 updt_s:0.754 data_s:0.752 smp/s:42 mem_gb:24.95
+Training:  57%|█████▋    | 34000/60000 [14:01:45<10:52:43,  1.51s/step]INFO 2026-07-22 16:09:55 ot_train.py:649 step:34K smpl:2M ep:49K epch:9.96 loss:0.035 grdn:0.308 lr:4.3e-05 updt_s:0.767 data_s:0.782 smp/s:41 mem_gb:24.93
+Training:  57%|█████▋    | 34200/60000 [14:06:52<10:52:58,  1.52s/step]INFO 2026-07-22 16:15:02 ot_train.py:649 step:34K smpl:2M ep:49K epch:10.02 loss:0.033 grdn:0.295 lr:4.3e-05 updt_s:0.762 data_s:0.769 smp/s:42 mem_gb:24.95
+Training:  57%|█████▋    | 34400/60000 [14:12:00<9:51:10,  1.39s/step] INFO 2026-07-22 16:20:09 ot_train.py:649 step:34K smpl:2M ep:50K epch:10.08 loss:0.032 grdn:0.290 lr:4.2e-05 updt_s:0.750 data_s:0.784 smp/s:42 mem_gb:24.95
+Training:  58%|█████▊    | 34600/60000 [14:16:08<8:43:24,  1.24s/step]INFO 2026-07-22 16:24:18 ot_train.py:649 step:35K smpl:2M ep:50K epch:10.14 loss:0.033 grdn:0.310 lr:4.2e-05 updt_s:0.646 data_s:0.593 smp/s:52 mem_gb:24.95
+Training:  58%|█████▊    | 34800/60000 [14:20:33<8:46:56,  1.25s/step]INFO 2026-07-22 16:28:42 ot_train.py:649 step:35K smpl:2M ep:50K epch:10.20 loss:0.034 grdn:0.303 lr:4.1e-05 updt_s:0.658 data_s:0.661 smp/s:49 mem_gb:24.95
+Training:  58%|█████▊    | 35000/60000 [14:25:40<8:45:42,  1.26s/step]INFO 2026-07-22 16:33:50 ot_train.py:649 step:35K smpl:2M ep:51K epch:10.26 loss:0.033 grdn:0.291 lr:4.1e-05 updt_s:0.672 data_s:0.865 smp/s:42 mem_gb:24.95
+Training:  59%|█████▊    | 35200/60000 [14:30:01<8:31:39,  1.24s/step]INFO 2026-07-22 16:38:11 ot_train.py:649 step:35K smpl:2M ep:51K epch:10.32 loss:0.033 grdn:0.298 lr:4.0e-05 updt_s:0.668 data_s:0.633 smp/s:49 mem_gb:24.95
+Training:  59%|█████▉    | 35400/60000 [14:34:08<8:14:30,  1.21s/step]INFO 2026-07-22 16:42:17 ot_train.py:649 step:35K smpl:2M ep:51K epch:10.38 loss:0.032 grdn:0.290 lr:4.0e-05 updt_s:0.649 data_s:0.581 smp/s:52 mem_gb:24.95
+Training:  59%|█████▉    | 35600/60000 [14:38:14<8:27:21,  1.25s/step]INFO 2026-07-22 16:46:24 ot_train.py:649 step:36K smpl:2M ep:51K epch:10.43 loss:0.033 grdn:0.296 lr:3.9e-05 updt_s:0.650 data_s:0.579 smp/s:52 mem_gb:24.95
+Training:  60%|█████▉    | 35800/60000 [14:42:21<8:19:30,  1.24s/step]INFO 2026-07-22 16:50:30 ot_train.py:649 step:36K smpl:2M ep:52K epch:10.49 loss:0.032 grdn:0.287 lr:3.9e-05 updt_s:0.651 data_s:0.579 smp/s:52 mem_gb:24.95
+Training:  60%|██████    | 36000/60000 [14:46:25<8:23:06,  1.26s/step]INFO 2026-07-22 16:54:35 ot_train.py:649 step:36K smpl:2M ep:52K epch:10.55 loss:0.031 grdn:0.274 lr:3.8e-05 updt_s:0.652 data_s:0.567 smp/s:52 mem_gb:24.95
+Training:  60%|██████    | 36200/60000 [14:50:31<8:09:53,  1.24s/step]INFO 2026-07-22 16:58:40 ot_train.py:649 step:36K smpl:2M ep:52K epch:10.61 loss:0.031 grdn:0.286 lr:3.7e-05 updt_s:0.654 data_s:0.572 smp/s:52 mem_gb:24.93
+Training:  61%|██████    | 36400/60000 [14:54:38<8:13:43,  1.26s/step]INFO 2026-07-22 17:02:48 ot_train.py:649 step:36K smpl:2M ep:53K epch:10.67 loss:0.031 grdn:0.297 lr:3.7e-05 updt_s:0.665 data_s:0.570 smp/s:52 mem_gb:24.95
+Training:  61%|██████    | 36600/60000 [14:58:43<7:48:04,  1.20s/step]INFO 2026-07-22 17:06:52 ot_train.py:649 step:37K smpl:2M ep:53K epch:10.73 loss:0.031 grdn:0.287 lr:3.6e-05 updt_s:0.649 data_s:0.571 smp/s:52 mem_gb:24.95
+Training:  61%|██████▏   | 36800/60000 [15:02:46<7:52:01,  1.22s/step]INFO 2026-07-22 17:10:55 ot_train.py:649 step:37K smpl:2M ep:53K epch:10.79 loss:0.029 grdn:0.269 lr:3.6e-05 updt_s:0.651 data_s:0.562 smp/s:53 mem_gb:24.95
+Training:  62%|██████▏   | 37000/60000 [15:06:47<7:31:09,  1.18s/step]INFO 2026-07-22 17:14:56 ot_train.py:649 step:37K smpl:2M ep:53K epch:10.84 loss:0.030 grdn:0.280 lr:3.5e-05 updt_s:0.644 data_s:0.560 smp/s:53 mem_gb:24.95
+Training:  62%|██████▏   | 37023/60000 [15:07:15<7:52:26,  1.23s/step]