Add files using upload-large-folder tool

Abs_6D/DiT_Layer/16/Baseline/baseline_gpu1_rolling5k_20260723_022922.log

@@ -0,0 +1,198 @@
+INFO 2026-07-23 02:29:36 ot_train.py:281 {'batch_size': 64,
+ 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000',
+ 'cudnn_deterministic': False,
+ 'dataset': {'depth_output_unit': 'mm',
+             'episodes': None,
+             'eval_split': 0.0,
+             'image_transforms': {'enable': False,
+                                  'max_num_transforms': 3,
+                                  'random_order': False,
+                                  'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
+                                                                            5.0],
+                                                                'translate': [0.05,
+                                                                              0.05]},
+                                                     'type': 'RandomAffine',
+                                                     'weight': 1.0},
+                                          'brightness': {'kwargs': {'brightness': [0.8,
+                                                                                   1.2]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'contrast': {'kwargs': {'contrast': [0.8,
+                                                                               1.2]},
+                                                       'type': 'ColorJitter',
+                                                       'weight': 1.0},
+                                          'hue': {'kwargs': {'hue': [-0.05,
+                                                                     0.05]},
+                                                  'type': 'ColorJitter',
+                                                  'weight': 1.0},
+                                          'saturation': {'kwargs': {'saturation': [0.5,
+                                                                                   1.5]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'sharpness': {'kwargs': {'sharpness': [0.5,
+                                                                                 1.5]},
+                                                        'type': 'SharpnessJitter',
+                                                        'weight': 1.0}}},
+             'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2',
+             'return_uint8': False,
+             'revision': None,
+             'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2',
+             'streaming': False,
+             'use_imagenet_stats': True,
+             'video_backend': 'torchcodec'},
+ 'env': None,
+ 'env_eval_freq': 20000,
+ 'eval': {'batch_size': 50,
+          'n_episodes': 50,
+          'recording': False,
+          'recording_private': False,
+          'recording_repo_id': None,
+          'use_async_envs': True},
+ 'eval_steps': 0,
+ 'job': {'detach': False,
+         'image': 'huggingface/lerobot-gpu:latest',
+         'tags': [],
+         'target': None,
+         'timeout': '2d'},
+ 'job_name': 'INSIGHT_6D_DiT16_Baseline',
+ 'log_freq': 200,
+ 'max_eval_samples': 0,
+ 'num_workers': 4,
+ 'optimizer': {'betas': [0.9, 0.999],
+               'eps': 1e-08,
+               'grad_clip_norm': 1.0,
+               'lr': 0.0001,
+               'type': 'adamw',
+               'weight_decay': 1e-05},
+ 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline',
+ 'peft': None,
+ 'persistent_workers': True,
+ 'policy': {'action_decode_transform': None,
+            'action_expert_num_layers': 16,
+            'balance_dataset_weights': True,
+            'balance_trajectory_weights': True,
+            'base_model_path': 'nvidia/GR00T-N1.7-3B',
+            'batch_size': 32,
+            'chunk_size': 16,
+            'dataloader_num_workers': 8,
+            'dataset_paths': None,
+            'device': 'cuda',
+            'embodiment_tag': 'new_embodiment',
+            'image_size': [256, 256],
+            'input_features': {'observation.images.guide': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.right_shoulder': {'shape': [3,
+                                                                               224,
+                                                                               224],
+                                                                     'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.wrist': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.state': {'shape': [16],
+                                                     'type': <FeatureType.STATE: 'STATE'>}},
+            'license': None,
+            'lora_alpha': 16,
+            'lora_dropout': 0.1,
+            'lora_full_model': False,
+            'lora_rank': 0,
+            'max_action_dim': 132,
+            'max_state_dim': 132,
+            'max_steps': 10000,
+            'model_params_fp32': True,
+            'n_action_steps': 16,
+            'n_obs_steps': 1,
+            'normalization_mapping': {'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
+            'num_inference_timesteps': None,
+            'optimizer_betas': [0.9, 0.999],
+            'optimizer_eps': 1e-08,
+            'optimizer_lr': 0.0001,
+            'optimizer_weight_decay': 1e-05,
+            'output_dir': './tmp/gr00t',
+            'output_features': {'action': {'shape': [10],
+                                           'type': <FeatureType.ACTION: 'ACTION'>}},
+            'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model',
+            'pretrained_revision': None,
+            'private': None,
+            'push_to_hub': False,
+            'relative_exclude_joints': [],
+            'repo_id': None,
+            'report_to': 'wandb',
+            'resume': False,
+            'rkd_angle_weight': 1.0,
+            'rkd_distance_weight': 1.0,
+            'rkd_enabled': False,
+            'rkd_fm_loss_weight': 1.0,
+            'rkd_loss_weight': 0.1,
+            'rkd_student_attention_hidden_dim': 512,
+            'rkd_student_attention_output_dim': 512,
+            'rkd_student_pooling': 'attention',
+            'rkd_vae_checkpoint': None,
+            'rtc_ramp_rate': None,
+            'save_steps': 1000,
+            'tags': None,
+            'tokenizer_assets_repo': None,
+            'tune_diffusion_model': True,
+            'tune_llm': False,
+            'tune_projector': True,
+            'tune_top_llm_layers': 0,
+            'tune_visual': False,
+            'tune_vlln': True,
+            'type': 'groot_rkd',
+            'use_amp': False,
+            'use_bf16': True,
+            'use_flash_attention': False,
+            'use_peft': False,
+            'use_relative_actions': False,
+            'video_backend': 'decord',
+            'warmup_ratio': 0.05,
+            'warmup_steps': None},
+ 'prefetch_factor': 4,
+ 'rename_map': {},
+ 'resume': True,
+ 'reward_model': None,
+ 'sample_weighting': None,
+ 'save_checkpoint': True,
+ 'save_checkpoint_to_hub': False,
+ 'save_freq': 20000,
+ 'scheduler': {'name': 'cosine', 'num_warmup_steps': 500, 'type': 'diffuser'},
+ 'seed': 42,
+ 'steps': 60000,
+ 'tolerance_s': 0.0001,
+ 'use_policy_training_preset': True,
+ 'wandb': {'add_tags': True,
+           'disable_artifact': True,
+           'enable': True,
+           'entity': None,
+           'mode': None,
+           'notes': None,
+           'project': 'lerobot',
+           'run_id': 'eb0sb786'}}
+Traceback (most recent call last):
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1613, in <module>
+    main()
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1609, in main
+    train_module.main()
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 822, in main
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 285, in train
+    if not policy.is_trainable:
+                   ^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/common/wandb_utils.py", line 99, in __init__
+    wandb.init(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1528, in init
+    get_sentry().reraise(e)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/analytics/sentry.py", line 191, in reraise
+    raise exc.with_traceback(tb)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1511, in init
+    run = wi.init(run_settings, run_config, run_printer)
+          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 961, in init
+    raise error
+wandb.errors.errors.UsageError: You provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming the run (eb0sb786) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`.

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/phase_schedule.json

@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "step_interval": "[start_step, until_step)",
+  "total_steps": 60000,
+  "optimizer_lifecycle": "single_continuous_optimizer",
+  "scheduler_lifecycle": "single_continuous_scheduler",
+  "phases": [
+    {
+      "index": 0,
+      "name": "rscl_rkd_cosine_decay_60K",
+      "start_step": 0,
+      "until_step": 60000,
+      "policy": {
+        "rkd_enabled": true,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_loss_weight_end": 0.0,
+        "rkd_loss_weight_schedule": "cosine",
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+      },
+      "trainable": {
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_top_llm_layers": 0,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/resolved_config.yaml

@@ -0,0 +1,35 @@
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  warmup_steps: 3000
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  rkd_enabled: true
+  rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11
+  rkd_student_pooling: attention
+  rkd_student_attention_hidden_dim: 512
+  rkd_student_attention_output_dim: 512
+  rkd_fm_loss_weight: 1.0
+  rkd_loss_weight: 1.0
+  rkd_distance_weight: 1.0
+  rkd_angle_weight: 2.0
+seed: 42
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+job_name: INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/source_config.yaml

@@ -0,0 +1,63 @@
+# INSIGHT GR00T N1.7 RKD training with RS-CL-style full-training regularization.
+# Common RKD values live in the top-level policy; the phase contains schedule overrides only.
+insight:
+  checkpoint_interval: 5K
+  checkpoint_slots: 2
+  # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
+  action_space: ee_abs_rot6d
+  # Options: coarse | detailed
+  prompt_set: detailed
+  save_steps: ["10K", "30K", "60K"]
+  phases:
+    - name: rscl_rkd_cosine_decay_60K
+      until_step: 60K
+      policy:
+        # Inherits RKD=1.0 below and follows a half-cosine decay to zero.
+        rkd_loss_weight_end: 0.0
+        rkd_loss_weight_schedule: cosine
+
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
+
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  # RS-CL GR00T uses exactly 3K warmup updates.
+  warmup_steps: 3000
+
+  # VLM
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  # Action Expert
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+
+  # Frozen TimewarpVAE teacher and common RKD objective.
+  rkd_enabled: true
+  rkd_vae_checkpoint: ${HOME}/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11
+  rkd_student_pooling: attention  # attention | mean
+  rkd_student_attention_hidden_dim: 512
+  rkd_student_attention_output_dim: 512
+  rkd_fm_loss_weight: 1.0
+  rkd_loss_weight: 1.0
+  rkd_distance_weight: 1.0
+  rkd_angle_weight: 2.0
+
+seed: 42
+# Single GPU/process, so this is also the global batch size.
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: ${HOME}/groot_insight/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+job_name: INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60
+
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/training_state/optimizer_param_groups.json

@@ -0,0 +1,361 @@
+[
+    {
+        "lr": 1.8865999845374793e-06,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 1.8865999845374793e-06,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310,
+            311,
+            312,
+            313,
+            314,
+            315,
+            316,
+            317,
+            318,
+            319,
+            320
+        ]
+    }
+]

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/training_state/training_step.json

@@ -0,0 +1,5 @@
+{
+    "step": 55000,
+    "num_processes": 1,
+    "batch_size": 64
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/phase_schedule.json

@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "step_interval": "[start_step, until_step)",
+  "total_steps": 60000,
+  "optimizer_lifecycle": "single_continuous_optimizer",
+  "scheduler_lifecycle": "single_continuous_scheduler",
+  "phases": [
+    {
+      "index": 0,
+      "name": "rscl_rkd_cosine_decay_60K",
+      "start_step": 0,
+      "until_step": 60000,
+      "policy": {
+        "rkd_enabled": true,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_loss_weight_end": 0.0,
+        "rkd_loss_weight_schedule": "cosine",
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+      },
+      "trainable": {
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_top_llm_layers": 0,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/config.json

@@ -0,0 +1,118 @@
+{
+    "type": "groot_rkd",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                16
+            ]
+        },
+        "observation.images.wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.right_shoulder": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.guide": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                10
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model",
+    "pretrained_revision": null,
+    "chunk_size": 16,
+    "n_action_steps": 16,
+    "max_state_dim": 132,
+    "max_action_dim": 132,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "IDENTITY",
+        "ACTION": "IDENTITY"
+    },
+    "base_model_path": "nvidia/GR00T-N1.7-3B",
+    "action_decode_transform": null,
+    "embodiment_tag": "new_embodiment",
+    "tune_llm": false,
+    "tune_visual": false,
+    "tune_projector": true,
+    "tune_diffusion_model": true,
+    "tune_vlln": true,
+    "tune_top_llm_layers": 0,
+    "num_inference_timesteps": null,
+    "rtc_ramp_rate": null,
+    "use_flash_attention": false,
+    "use_relative_actions": false,
+    "relative_exclude_joints": [],
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-05,
+    "warmup_steps": 3000,
+    "warmup_ratio": 0.05,
+    "use_bf16": true,
+    "model_params_fp32": true,
+    "image_size": [
+        256,
+        256
+    ],
+    "tokenizer_assets_repo": null,
+    "lora_rank": 0,
+    "lora_alpha": 16,
+    "lora_dropout": 0.1,
+    "lora_full_model": false,
+    "video_backend": "decord",
+    "balance_dataset_weights": true,
+    "balance_trajectory_weights": true,
+    "dataset_paths": null,
+    "output_dir": "./tmp/gr00t",
+    "save_steps": 1000,
+    "max_steps": 10000,
+    "batch_size": 32,
+    "dataloader_num_workers": 8,
+    "report_to": "wandb",
+    "resume": false,
+    "action_expert_num_layers": 16,
+    "rkd_enabled": true,
+    "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11",
+    "rkd_student_pooling": "attention",
+    "rkd_student_attention_hidden_dim": 512,
+    "rkd_student_attention_output_dim": 512,
+    "rkd_fm_loss_weight": 1.0,
+    "rkd_loss_weight": 1.0,
+    "rkd_distance_weight": 1.0,
+    "rkd_angle_weight": 2.0
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/phase_schedule.json

@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "step_interval": "[start_step, until_step)",
+  "total_steps": 60000,
+  "optimizer_lifecycle": "single_continuous_optimizer",
+  "scheduler_lifecycle": "single_continuous_scheduler",
+  "phases": [
+    {
+      "index": 0,
+      "name": "rscl_rkd_cosine_decay_60K",
+      "start_step": 0,
+      "until_step": 60000,
+      "policy": {
+        "rkd_enabled": true,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_loss_weight_end": 0.0,
+        "rkd_loss_weight_schedule": "cosine",
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+      },
+      "trainable": {
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_top_llm_layers": 0,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/train_config.json

@@ -0,0 +1,269 @@
+{
+    "dataset": {
+        "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+        "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "groot_rkd",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    16
+                ]
+            },
+            "observation.images.wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_shoulder": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.guide": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    10
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model",
+        "pretrained_revision": null,
+        "chunk_size": 16,
+        "n_action_steps": 16,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "IDENTITY",
+            "ACTION": "IDENTITY"
+        },
+        "base_model_path": "nvidia/GR00T-N1.7-3B",
+        "action_decode_transform": null,
+        "embodiment_tag": "new_embodiment",
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true,
+        "tune_top_llm_layers": 0,
+        "num_inference_timesteps": null,
+        "rtc_ramp_rate": null,
+        "use_flash_attention": false,
+        "use_relative_actions": false,
+        "relative_exclude_joints": [],
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-05,
+        "warmup_steps": 3000,
+        "warmup_ratio": 0.05,
+        "use_bf16": true,
+        "model_params_fp32": true,
+        "image_size": [
+            256,
+            256
+        ],
+        "tokenizer_assets_repo": null,
+        "lora_rank": 0,
+        "lora_alpha": 16,
+        "lora_dropout": 0.1,
+        "lora_full_model": false,
+        "video_backend": "decord",
+        "balance_dataset_weights": true,
+        "balance_trajectory_weights": true,
+        "dataset_paths": null,
+        "output_dir": "./tmp/gr00t",
+        "save_steps": 1000,
+        "max_steps": 10000,
+        "batch_size": 32,
+        "dataloader_num_workers": 8,
+        "report_to": "wandb",
+        "resume": false,
+        "action_expert_num_layers": 16,
+        "rkd_enabled": true,
+        "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11",
+        "rkd_student_pooling": "attention",
+        "rkd_student_attention_hidden_dim": 512,
+        "rkd_student_attention_output_dim": 512,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+    },
+    "reward_model": null,
+    "output_dir": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K",
+    "job_name": "INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60",
+    "resume": true,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 64,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 60000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-05,
+        "grad_clip_norm": 1.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 3000,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "rs1egwv8",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01"
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/resolved_config.yaml

@@ -0,0 +1,35 @@
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  warmup_steps: 3000
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  rkd_enabled: true
+  rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11
+  rkd_student_pooling: attention
+  rkd_student_attention_hidden_dim: 512
+  rkd_student_attention_output_dim: 512
+  rkd_fm_loss_weight: 1.0
+  rkd_loss_weight: 1.0
+  rkd_distance_weight: 1.0
+  rkd_angle_weight: 2.0
+seed: 42
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+job_name: INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/source_config.yaml

@@ -0,0 +1,63 @@
+# INSIGHT GR00T N1.7 RKD training with RS-CL-style full-training regularization.
+# Common RKD values live in the top-level policy; the phase contains schedule overrides only.
+insight:
+  checkpoint_interval: 5K
+  checkpoint_slots: 2
+  # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
+  action_space: ee_abs_rot6d
+  # Options: coarse | detailed
+  prompt_set: detailed
+  save_steps: ["10K", "30K", "60K"]
+  phases:
+    - name: rscl_rkd_cosine_decay_60K
+      until_step: 60K
+      policy:
+        # Inherits RKD=1.0 below and follows a half-cosine decay to zero.
+        rkd_loss_weight_end: 0.0
+        rkd_loss_weight_schedule: cosine
+
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
+
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  # RS-CL GR00T uses exactly 3K warmup updates.
+  warmup_steps: 3000
+
+  # VLM
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  # Action Expert
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+
+  # Frozen TimewarpVAE teacher and common RKD objective.
+  rkd_enabled: true
+  rkd_vae_checkpoint: ${HOME}/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11
+  rkd_student_pooling: attention  # attention | mean
+  rkd_student_attention_hidden_dim: 512
+  rkd_student_attention_output_dim: 512
+  rkd_fm_loss_weight: 1.0
+  rkd_loss_weight: 1.0
+  rkd_distance_weight: 1.0
+  rkd_angle_weight: 2.0
+
+seed: 42
+# Single GPU/process, so this is also the global batch size.
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: ${HOME}/groot_insight/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+job_name: INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60
+
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/training_state/optimizer_param_groups.json

@@ -0,0 +1,361 @@
+[
+    {
+        "lr": 0.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154,
+            155,
+            156,
+            157,
+            158
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 0.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310,
+            311,
+            312,
+            313,
+            314,
+            315,
+            316,
+            317,
+            318,
+            319,
+            320
+        ]
+    }
+]

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/rscl_gpu1_rolling5k_20260723_022922.log

@@ -0,0 +1,198 @@
+INFO 2026-07-23 02:29:37 ot_train.py:281 {'batch_size': 64,
+ 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000',
+ 'cudnn_deterministic': False,
+ 'dataset': {'depth_output_unit': 'mm',
+             'episodes': None,
+             'eval_split': 0.0,
+             'image_transforms': {'enable': False,
+                                  'max_num_transforms': 3,
+                                  'random_order': False,
+                                  'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
+                                                                            5.0],
+                                                                'translate': [0.05,
+                                                                              0.05]},
+                                                     'type': 'RandomAffine',
+                                                     'weight': 1.0},
+                                          'brightness': {'kwargs': {'brightness': [0.8,
+                                                                                   1.2]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'contrast': {'kwargs': {'contrast': [0.8,
+                                                                               1.2]},
+                                                       'type': 'ColorJitter',
+                                                       'weight': 1.0},
+                                          'hue': {'kwargs': {'hue': [-0.05,
+                                                                     0.05]},
+                                                  'type': 'ColorJitter',
+                                                  'weight': 1.0},
+                                          'saturation': {'kwargs': {'saturation': [0.5,
+                                                                                   1.5]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'sharpness': {'kwargs': {'sharpness': [0.5,
+                                                                                 1.5]},
+                                                        'type': 'SharpnessJitter',
+                                                        'weight': 1.0}}},
+             'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2',
+             'return_uint8': False,
+             'revision': None,
+             'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2',
+             'streaming': False,
+             'use_imagenet_stats': True,
+             'video_backend': 'torchcodec'},
+ 'env': None,
+ 'env_eval_freq': 20000,
+ 'eval': {'batch_size': 50,
+          'n_episodes': 50,
+          'recording': False,
+          'recording_private': False,
+          'recording_repo_id': None,
+          'use_async_envs': True},
+ 'eval_steps': 0,
+ 'job': {'detach': False,
+         'image': 'huggingface/lerobot-gpu:latest',
+         'tags': [],
+         'target': None,
+         'timeout': '2d'},
+ 'job_name': 'INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60',
+ 'log_freq': 200,
+ 'max_eval_samples': 0,
+ 'num_workers': 4,
+ 'optimizer': {'betas': [0.9, 0.999],
+               'eps': 1e-08,
+               'grad_clip_norm': 1.0,
+               'lr': 0.0001,
+               'type': 'adamw',
+               'weight_decay': 1e-05},
+ 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K',
+ 'peft': None,
+ 'persistent_workers': True,
+ 'policy': {'action_decode_transform': None,
+            'action_expert_num_layers': 16,
+            'balance_dataset_weights': True,
+            'balance_trajectory_weights': True,
+            'base_model_path': 'nvidia/GR00T-N1.7-3B',
+            'batch_size': 32,
+            'chunk_size': 16,
+            'dataloader_num_workers': 8,
+            'dataset_paths': None,
+            'device': 'cuda',
+            'embodiment_tag': 'new_embodiment',
+            'image_size': [256, 256],
+            'input_features': {'observation.images.guide': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.right_shoulder': {'shape': [3,
+                                                                               224,
+                                                                               224],
+                                                                     'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.wrist': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.state': {'shape': [16],
+                                                     'type': <FeatureType.STATE: 'STATE'>}},
+            'license': None,
+            'lora_alpha': 16,
+            'lora_dropout': 0.1,
+            'lora_full_model': False,
+            'lora_rank': 0,
+            'max_action_dim': 132,
+            'max_state_dim': 132,
+            'max_steps': 10000,
+            'model_params_fp32': True,
+            'n_action_steps': 16,
+            'n_obs_steps': 1,
+            'normalization_mapping': {'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
+            'num_inference_timesteps': None,
+            'optimizer_betas': [0.9, 0.999],
+            'optimizer_eps': 1e-08,
+            'optimizer_lr': 0.0001,
+            'optimizer_weight_decay': 1e-05,
+            'output_dir': './tmp/gr00t',
+            'output_features': {'action': {'shape': [10],
+                                           'type': <FeatureType.ACTION: 'ACTION'>}},
+            'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model',
+            'pretrained_revision': None,
+            'private': None,
+            'push_to_hub': False,
+            'relative_exclude_joints': [],
+            'repo_id': None,
+            'report_to': 'wandb',
+            'resume': False,
+            'rkd_angle_weight': 2.0,
+            'rkd_distance_weight': 1.0,
+            'rkd_enabled': True,
+            'rkd_fm_loss_weight': 1.0,
+            'rkd_loss_weight': 1.0,
+            'rkd_student_attention_hidden_dim': 512,
+            'rkd_student_attention_output_dim': 512,
+            'rkd_student_pooling': 'attention',
+            'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11',
+            'rtc_ramp_rate': None,
+            'save_steps': 1000,
+            'tags': None,
+            'tokenizer_assets_repo': None,
+            'tune_diffusion_model': True,
+            'tune_llm': False,
+            'tune_projector': True,
+            'tune_top_llm_layers': 0,
+            'tune_visual': False,
+            'tune_vlln': True,
+            'type': 'groot_rkd',
+            'use_amp': False,
+            'use_bf16': True,
+            'use_flash_attention': False,
+            'use_peft': False,
+            'use_relative_actions': False,
+            'video_backend': 'decord',
+            'warmup_ratio': 0.05,
+            'warmup_steps': 3000},
+ 'prefetch_factor': 4,
+ 'rename_map': {},
+ 'resume': True,
+ 'reward_model': None,
+ 'sample_weighting': None,
+ 'save_checkpoint': True,
+ 'save_checkpoint_to_hub': False,
+ 'save_freq': 20000,
+ 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'},
+ 'seed': 42,
+ 'steps': 60000,
+ 'tolerance_s': 0.0001,
+ 'use_policy_training_preset': True,
+ 'wandb': {'add_tags': True,
+           'disable_artifact': True,
+           'enable': True,
+           'entity': None,
+           'mode': None,
+           'notes': None,
+           'project': 'lerobot',
+           'run_id': 'cemf0rge'}}
+Traceback (most recent call last):
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1613, in <module>
+    main()
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1609, in main
+    train_module.main()
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 825, in main
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 285, in train
+    if not policy.is_trainable:
+                   ^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/common/wandb_utils.py", line 99, in __init__
+    wandb.init(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1528, in init
+    get_sentry().reraise(e)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/analytics/sentry.py", line 191, in reraise
+    raise exc.with_traceback(tb)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1511, in init
+    run = wi.init(run_settings, run_config, run_printer)
+          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 961, in init
+    raise error
+wandb.errors.errors.UsageError: You provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming the run (cemf0rge) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`.

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/source_config.yaml

@@ -1,6 +1,8 @@
 # INSIGHT GR00T N1.7 RKD training with RS-CL-style full-training regularization.
 # Common RKD values live in the top-level policy; the phase contains schedule overrides only.
 insight:
+  checkpoint_interval: 5K
+  checkpoint_slots: 2
   # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
   action_space: ee_abs_rot6d
   # Options: coarse | detailed

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/debug.log

@@ -1,20 +1,65 @@
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_setup.py:_flush():81] Configure stats pid to 814719
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/logs/debug.log
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/logs/debug-internal.log
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_init.py:init():766] calling init triggers
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
-config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'pgr3eari', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}}
-2026-07-23 01:33:42,299 INFO    MainThread:814719 [wandb_init.py:init():814] starting backend
-2026-07-23 01:33:42,517 INFO    MainThread:814719 [wandb_init.py:init():829] sending inform_init request
-2026-07-23 01:33:42,889 INFO    MainThread:814719 [wandb_init.py:init():834] backend started and connected
-2026-07-23 01:33:42,891 INFO    MainThread:814719 [wandb_init.py:init():904] updated telemetry
-2026-07-23 01:33:42,896 INFO    MainThread:814719 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
-2026-07-23 01:33:43,638 INFO    MainThread:814719 [wandb_init.py:init():967] run resumed
-2026-07-23 01:33:43,640 INFO    MainThread:814719 [wandb_init.py:init():972] starting run threads in backend
-2026-07-23 01:33:43,704 INFO    MainThread:814719 [wandb_run.py:_console_start():2523] atexit reg
-2026-07-23 01:33:43,704 INFO    MainThread:814719 [wandb_run.py:_redirect():2373] redirect: wrap_raw
-2026-07-23 01:33:43,705 INFO    MainThread:814719 [wandb_run.py:_redirect():2442] Wrapping output streams.
-2026-07-23 01:33:43,705 INFO    MainThread:814719 [wandb_run.py:_redirect():2465] Redirects installed.
-2026-07-23 01:33:43,706 INFO    MainThread:814719 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_setup.py:_flush():81] Configure stats pid to 1459
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/logs/debug.log
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/logs/debug-internal.log
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:init():766] calling init triggers
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'rs1egwv8', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01', '_wandb': {}}
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:init():814] starting backend
+2026-07-23 16:35:01,833 INFO    MainThread:1459 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 16:35:02,323 INFO    MainThread:1459 [wandb_init.py:init():834] backend started and connected
+2026-07-23 16:35:02,325 INFO    MainThread:1459 [wandb_init.py:init():904] updated telemetry
+2026-07-23 16:35:02,326 INFO    MainThread:1459 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 16:35:03,342 INFO    MainThread:1459 [wandb_init.py:init():967] run resumed
+2026-07-23 16:35:03,344 INFO    MainThread:1459 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 16:35:03,398 INFO    MainThread:1459 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-24 04:10:06,982 INFO    wandb-AsyncioManager-main:1459 [service_client.py:_forward_responses():122] Reached EOF.
+2026-07-24 04:10:06,983 INFO    wandb-AsyncioManager-main:1459 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
+2026-07-24 04:10:06,983 ERROR   wandb-AsyncioManager-main:1459 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
+    await asyncio_compat.race(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
+    async with open_task_group(race=True) as tg:
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
+    await anext(self.gen)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
+    await task_group._wait_all(race=race, timeout=exit_timeout)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
+    raise exc
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
+    result = await handle.wait_async(timeout=None)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
+    response = await self._handle.wait_async(timeout=timeout)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
+    raise HandleAbandonedError()
+wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
+2026-07-24 04:10:14,759 ERROR   wandb-AsyncioManager-main:1459 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
+    should_stop = await self._check_should_stop()
+                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
+    handle = await self._interface.deliver_async(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
+    handle = await self._client.deliver(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
+    handle = self._mailbox.require_response(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
+    raise MailboxClosedError()
+wandb.sdk.mailbox.mailbox.MailboxClosedError

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/files/config.yaml

@@ -0,0 +1,334 @@
+_wandb:
+    value:
+        cli_version: 0.27.2
+        e:
+            vmsr43xlhcpoc4wtuae54k6a709gs2et:
+                args:
+                    - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/train_config.json
+                    - --resume=true
+                    - --output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+                    - --dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+                codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                codePathLocal: my_scripts/lerobot_train_yaml.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "1888425144320"
+                        used: "1137071321088"
+                executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python
+                git:
+                    commit: a329728c2c9524d79b966c4a7a82ee56a4c2abf7
+                    remote: git@github.com:paragon7060/clvla.git
+                gpu: NVIDIA H200
+                gpu_count: 2
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc
+                host: ext_minje-box
+                memory:
+                    total: "2164176568320"
+                os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39
+                program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                python: CPython 3.12.13
+                root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+                startedAt: "2026-07-23T01:33:42.298193Z"
+                writerId: vmsr43xlhcpoc4wtuae54k6a709gs2et
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.12.13
+            "5": 0.27.2
+            "6": 5.5.4
+            "10":
+                - 21
+            "12": 0.27.2
+            "13": linux-x86_64
+batch_size:
+    value: 64
+checkpoint_path:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000
+cudnn_deterministic:
+    value: false
+dataset:
+    value:
+        depth_output_unit: mm
+        episodes: null
+        eval_split: 0
+        image_transforms:
+            enable: false
+            max_num_transforms: 3
+            random_order: false
+            tfs:
+                affine:
+                    kwargs:
+                        degrees:
+                            - -5
+                            - 5
+                        translate:
+                            - 0.05
+                            - 0.05
+                    type: RandomAffine
+                    weight: 1
+                brightness:
+                    kwargs:
+                        brightness:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                contrast:
+                    kwargs:
+                        contrast:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                hue:
+                    kwargs:
+                        hue:
+                            - -0.05
+                            - 0.05
+                    type: ColorJitter
+                    weight: 1
+                saturation:
+                    kwargs:
+                        saturation:
+                            - 0.5
+                            - 1.5
+                    type: ColorJitter
+                    weight: 1
+                sharpness:
+                    kwargs:
+                        sharpness:
+                            - 0.5
+                            - 1.5
+                    type: SharpnessJitter
+                    weight: 1
+        repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+        return_uint8: false
+        revision: null
+        root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+        streaming: false
+        use_imagenet_stats: true
+        video_backend: torchcodec
+env:
+    value: null
+env_eval_freq:
+    value: 20000
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        recording: false
+        recording_private: false
+        recording_repo_id: null
+        use_async_envs: true
+eval_steps:
+    value: 0
+job:
+    value:
+        detach: false
+        image: huggingface/lerobot-gpu:latest
+        tags: []
+        target: null
+        timeout: 2d
+job_name:
+    value: INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60
+log_freq:
+    value: 200
+max_eval_samples:
+    value: 0
+num_workers:
+    value: 4
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.999
+        eps: 1e-08
+        grad_clip_norm: 1
+        lr: 0.0001
+        type: adamw
+        weight_decay: 1e-05
+output_dir:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+peft:
+    value: null
+persistent_workers:
+    value: true
+policy:
+    value:
+        action_decode_transform: null
+        action_expert_num_layers: 16
+        balance_dataset_weights: true
+        balance_trajectory_weights: true
+        base_model_path: nvidia/GR00T-N1.7-3B
+        batch_size: 32
+        chunk_size: 16
+        dataloader_num_workers: 8
+        dataset_paths: null
+        device: cuda
+        embodiment_tag: new_embodiment
+        image_size:
+            - 256
+            - 256
+        input_features:
+            observation.images.guide:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.right_shoulder:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.wrist:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.state:
+                shape:
+                    - 16
+                type: STATE
+        license: null
+        lora_alpha: 16
+        lora_dropout: 0.1
+        lora_full_model: false
+        lora_rank: 0
+        max_action_dim: 132
+        max_state_dim: 132
+        max_steps: 10000
+        model_params_fp32: true
+        n_action_steps: 16
+        n_obs_steps: 1
+        normalization_mapping:
+            ACTION: IDENTITY
+            STATE: IDENTITY
+            VISUAL: IDENTITY
+        num_inference_timesteps: null
+        optimizer_betas:
+            - 0.9
+            - 0.999
+        optimizer_eps: 1e-08
+        optimizer_lr: 0.0001
+        optimizer_weight_decay: 1e-05
+        output_dir: ./tmp/gr00t
+        output_features:
+            action:
+                shape:
+                    - 10
+                type: ACTION
+        pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model
+        pretrained_revision: null
+        private: null
+        push_to_hub: false
+        relative_exclude_joints: []
+        repo_id: null
+        report_to: wandb
+        resume: false
+        rkd_angle_weight: 2
+        rkd_distance_weight: 1
+        rkd_enabled: true
+        rkd_fm_loss_weight: 1
+        rkd_loss_weight: 1
+        rkd_student_attention_hidden_dim: 512
+        rkd_student_attention_output_dim: 512
+        rkd_student_pooling: attention
+        rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11
+        rtc_ramp_rate: null
+        save_steps: 1000
+        tags: null
+        tokenizer_assets_repo: null
+        tune_diffusion_model: true
+        tune_llm: false
+        tune_projector: true
+        tune_top_llm_layers: 0
+        tune_visual: false
+        tune_vlln: true
+        type: groot_rkd
+        use_amp: false
+        use_bf16: true
+        use_flash_attention: false
+        use_peft: false
+        use_relative_actions: false
+        video_backend: decord
+        warmup_ratio: 0.05
+        warmup_steps: 3000
+prefetch_factor:
+    value: 4
+resume:
+    value: true
+reward_model:
+    value: null
+sample_weighting:
+    value: null
+save_checkpoint:
+    value: true
+save_checkpoint_to_hub:
+    value: false
+save_freq:
+    value: 20000
+scheduler:
+    value:
+        name: cosine
+        num_warmup_steps: 3000
+        type: diffuser
+seed:
+    value: 42
+steps:
+    value: 60000
+tolerance_s:
+    value: 0.0001
+use_policy_training_preset:
+    value: true
+wandb:
+    value:
+        add_tags: true
+        disable_artifact: true
+        enable: true
+        entity: null
+        mode: null
+        notes: null
+        project: lerobot
+        run_id: pgr3eari

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/files/output.log

@@ -45,4 +45,47 @@ Training:   3%|▎         | 800/30000 [20:31<12:21:37,  1.52s/step]INFO 2026-07
 Training:   3%|▎         | 1000/30000 [26:18<22:10:00,  2.75s/step]INFO 2026-07-23 02:00:30 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.09 loss:0.039 grdn:0.298 lr:5.2e-05 updt_s:0.843 data_s:0.886 smp/s:37 mem_gb:29.84
 Training:   4%|▍         | 1200/30000 [31:32<10:05:29,  1.26s/step]INFO 2026-07-23 02:05:45 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.14 loss:0.039 grdn:0.306 lr:5.1e-05 updt_s:0.805 data_s:0.765 smp/s:41 mem_gb:29.84
 Training:   5%|▍         | 1400/30000 [36:06<10:52:36,  1.37s/step]INFO 2026-07-23 02:10:19 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.20 loss:0.039 grdn:0.298 lr:5.1e-05 updt_s:0.744 data_s:0.622 smp/s:47 mem_gb:29.84
-Training:   5%|▍         | 1482/30000 [38:04<10:54:48,  1.38s/step]
+Training:   5%|▌         | 1600/30000 [41:01<19:58:58,  2.53s/step]INFO 2026-07-23 02:15:14 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.26 loss:0.039 grdn:0.299 lr:5.0e-05 updt_s:0.806 data_s:0.666 smp/s:43 mem_gb:29.82
+Training:   6%|▌         | 1800/30000 [45:51<10:43:33,  1.37s/step]INFO 2026-07-23 02:20:04 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.32 loss:0.037 grdn:0.292 lr:4.9e-05 updt_s:0.787 data_s:0.659 smp/s:44 mem_gb:29.85
+Training:   7%|▋         | 2000/30000 [50:39<10:41:47,  1.38s/step]INFO 2026-07-23 02:24:51 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.38 loss:0.037 grdn:0.286 lr:4.9e-05 updt_s:0.776 data_s:0.660 smp/s:45 mem_gb:29.85
+Training:   7%|▋         | 2167/30000 [54:37<10:34:57,  1.37s/step]Traceback (most recent call last):
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1453, in <module>
+    config_path = Path(config_value).expanduser()
+    ^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main
+    config_value = _find_cli_value(args, "--config_path")
+    ^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 618, in train
+    if is_eval_step:
+
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 179, in update_policy
+    """
+
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot_rkd.py", line 326, in forward
+    outputs = self._groot_model.forward(groot_inputs)
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/groot_n1_7.py", line 884, in forward
+    backbone_outputs = self.backbone(backbone_inputs)
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/groot_n1_7.py", line 450, in forward
+    self._ensure_legacy_qwen3_position_ids(model_input)
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/groot_n1_7.py", line 402, in _ensure_legacy_qwen3_position_ids
+    position_ids = compute_3d_position_ids(
+                   ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/models/qwen3_vl/modeling_qwen3_vl.py", line 1228, in compute_3d_position_ids
+    position_ids, rope_deltas = self.get_rope_index(
+                                ^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/models/qwen3_vl/modeling_qwen3_vl.py", line 1106, in get_rope_index
+    current_pos += text_len
+    ^^^^^^^^^^^
+KeyboardInterrupt

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/rkd_teacher_latent_norm":1,"train/weighted_rkd_loss":0.001385301467962563,"_timestamp":1.7847403914008305e+09,"train/training_phase_start_step":0,"train/rkd_loss_weight":0.3208404665513721,"_wandb":{"runtime":59006},"train/weighted_flow_matching_loss":0.022767670452594757,"train/episodes":53461.55783611993,"train/grad_norm":0.26664105713367464,"train/rkd_distance_loss":0.0010141097009181976,"train/training_phase_trainable_params":1.080755329e+09,"train/gpu_mem_gb":29.854091150760652,"train/flow_matching_loss":0.022767670452594757,"train/rkd_student_feature_norm":16.188758850097656,"train/rkd_runtime_enabled":1,"train/loss":0.024152971804142,"train/samples_per_s":49.78324781714454,"train/lr":3.5336255062138e-05,"train/steps":37000,"_step":37000,"train/rkd_angle_loss":0.0016518083866685629,"train/samples":2368000,"train/rkd_loss":0.004317726474255323,"train/training_phase_until_step":60000,"train/dataloading_s":0.5792085527597374,"_runtime":59006,"train/training_phase_index":0,"train/rkd_fm_loss_weight":1,"train/update_s":0.7063644623853179,"train/rkd_student_pooling_id":1,"train/epochs":10.844129378523313}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/logs/debug-core.log

@@ -26,3 +26,39 @@
 {"time":"2026-07-23T01:33:48.72926561Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"0uuooyt4of4j"}
 {"time":"2026-07-23T01:33:48.824026381Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"v9b9occvkv0z"}
 {"time":"2026-07-23T01:33:48.928044163Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3mgxscqyzkyn"}
+{"time":"2026-07-23T02:28:50.768484688Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768650257Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768660338Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768702111Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768725726Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768737691Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.768728092Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.768751955Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768745569Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768805358Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768815828Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814707-815594-198914184/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:50.76883043Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814713-815525-1635480438/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:50.770084447Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.770158191Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.770173329Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.770146502Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.770246375Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814719-815551-3138903381/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:50.770382899Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773003008Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773054953Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773111864Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773106808Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773073579Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.773200905Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814701-815561-915654594/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:52.626908382Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.626929775Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.626940482Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:28:52.662753611Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.662779647Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.662788745Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:28:52.68387658Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.683943364Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.683953617Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:29:02.470789399Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:29:02.470826347Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:29:02.470833873Z","level":"INFO","msg":"server is closed"}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/logs/debug-internal.log

@@ -245,3 +245,109 @@
 {"time":"2026-07-23T02:11:32.739406463Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":300,"events_lines":2,"console_offset":47,"console_lines":1}
 {"time":"2026-07-23T02:12:22.402029921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-07-23T02:12:22.4021656Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T02:12:23.362164613Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:12:23.362395596Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":302,"events_lines":6,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:12:24.144626493Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:12:24.144870858Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:13:12.098678344Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:13:12.099003095Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":308,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:13:12.458838875Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:13:12.459121578Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":310,"events_lines":4,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:04.269679981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:04.269842494Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:04.959605431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:04.959869932Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":314,"events_lines":8,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:47.260819768Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:47.261088178Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:47.816232955Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:47.816499019Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":322,"events_lines":6,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:15:14.209850024Z","level":"WARN","msg":"handler: ignoring partial history record","step":31600,"current":37001}
+{"time":"2026-07-23T02:15:23.521072135Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:15:23.521353534Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:15:23.982612715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:15:23.982826206Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":328,"events_lines":4,"console_offset":47,"console_lines":2}
+{"time":"2026-07-23T02:15:24.425232875Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:15:24.425483013Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:16:01.786406101Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:16:01.786603022Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":332,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:16:25.713166877Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:16:25.713431331Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":334,"events_lines":4,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:17:03.266808763Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:17:03.267087671Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":338,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:17:50.530237524Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:17:50.530495713Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":340,"events_lines":6,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:18:20.451787991Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:18:20.452026414Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":346,"events_lines":6,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:01.788519665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:01.788703845Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":352,"events_lines":4,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:02.328045556Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:02.328268427Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":356,"events_lines":6,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:02.759022028Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:02.759194577Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:03.248435965Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:16.837240643Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":362,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:17.303817335Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:31.836655244Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":364,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:20:04.018094282Z","level":"WARN","msg":"handler: ignoring partial history record","step":31800,"current":37001}
+{"time":"2026-07-23T02:20:13.742987022Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:13.743151354Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T02:20:14.186936411Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:14.187196098Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":366,"events_lines":5,"console_offset":48,"console_lines":2}
+{"time":"2026-07-23T02:20:14.606163567Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:14.606358638Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":371,"events_lines":1}
+{"time":"2026-07-23T02:20:47.648537651Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:47.648719247Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":372,"events_lines":1,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:20:48.12626836Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:48.126523688Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":373,"events_lines":3,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:21:12.996289747Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:21:12.996538083Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":376,"events_lines":1,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:21:40.076672259Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:21:40.076833315Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":377,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:22:12.591294436Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:22:12.591534984Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":379,"events_lines":3,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:22:13.008257666Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:22:13.008448544Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":382,"events_lines":4,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:22:43.455358315Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:22:43.4555333Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:23:15.177406156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:23:15.177600701Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":386,"events_lines":4,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:23:15.790816852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:23:15.791043145Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":390,"events_lines":6,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:23:16.511965589Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:23:16.531664799Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:24:09.59999693Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:09.600190154Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T02:24:10.096383571Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:10.096565421Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":396,"events_lines":6,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:24:10.56096879Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:10.87257939Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:24:51.945277879Z","level":"WARN","msg":"handler: ignoring partial history record","step":32000,"current":37001}
+{"time":"2026-07-23T02:24:53.714569624Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:53.714700687Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-07-23T02:24:54.093680458Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:54.093909985Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":402,"events_lines":6,"console_offset":49,"console_lines":2}
+{"time":"2026-07-23T02:25:49.372929793Z","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/minje227_hyu-hanyang-university/lerobot/pgr3eari/file_stream","body":"{\"error\":\"context deadline exceeded\"}"}
+{"time":"2026-07-23T02:26:24.414750595Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:26:24.414969006Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:04.885943619Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:04.886281642Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":408,"events_lines":12,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:05.35140455Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:05.351607052Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":420,"events_lines":6,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:31.046145432Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:31.046284597Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:58.5330972Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:58.533277753Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":426,"events_lines":4,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:58.938590054Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:58.938897544Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":430,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:28:32.127453304Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:28:32.127626571Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":432,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:28:52.097937416Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-07-23T02:29:01.470938231Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:29:01.471166513Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":434,"events_lines":4,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:29:01.947602559Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:29:01.947868498Z","level":"INFO","msg":"filestream: sending request","total_files":3,"events_offset":438,"events_lines":2,"console_offset":50,"console_lines":41,"uploaded_len":3,"complete":true,"exit_code":255}
+{"time":"2026-07-23T02:29:02.470009043Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:29:02.470189702Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-07-23T02:29:02.470200334Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-07-23T02:29:02.470273871Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-07-23T02:29:02.470277743Z","level":"INFO","msg":"stream: all finished"}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_013342-pgr3eari/logs/debug.log

@@ -18,3 +18,48 @@ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2'
 2026-07-23 01:33:43,705 INFO    MainThread:814719 [wandb_run.py:_redirect():2442] Wrapping output streams.
 2026-07-23 01:33:43,705 INFO    MainThread:814719 [wandb_run.py:_redirect():2465] Redirects installed.
 2026-07-23 01:33:43,706 INFO    MainThread:814719 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-23 02:28:50,770 INFO    wandb-AsyncioManager-main:814719 [service_client.py:_forward_responses():122] Reached EOF.
+2026-07-23 02:28:50,770 INFO    wandb-AsyncioManager-main:814719 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
+2026-07-23 02:28:50,770 ERROR   wandb-AsyncioManager-main:814719 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
+    await asyncio_compat.race(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
+    async with open_task_group(race=True) as tg:
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
+    await anext(self.gen)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
+    await task_group._wait_all(race=race, timeout=exit_timeout)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
+    raise exc
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
+    result = await handle.wait_async(timeout=None)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
+    response = await self._handle.wait_async(timeout=timeout)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
+    raise HandleAbandonedError()
+wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
+2026-07-23 02:28:59,234 ERROR   wandb-AsyncioManager-main:814719 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
+    should_stop = await self._check_should_stop()
+                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
+    handle = await self._interface.deliver_async(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
+    handle = await self._client.deliver(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
+    handle = self._mailbox.require_response(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
+    raise MailboxClosedError()
+wandb.sdk.mailbox.mailbox.MailboxClosedError

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_022937-cemf0rge/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0},"_runtime":0}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_022937-cemf0rge/logs/debug-core.log

@@ -0,0 +1,30 @@
+{"time":"2026-07-23T02:29:37.218272849Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpbi3491l3/port-2499418.txt","pid":2499418,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T02:29:37.218902899Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2499418}
+{"time":"2026-07-23T02:29:37.218864854Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2499418-2500321-1544228546/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:37.402297125Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T02:29:37.409066123Z","level":"INFO","msg":"handleInformInit: received","streamId":"eb0sb786","id":"1(@)"}
+{"time":"2026-07-23T02:29:37.868916316Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"eb0sb786","id":"1(@)"}
+{"time":"2026-07-23T02:29:37.870710243Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpewnffvrr/port-2499551.txt","pid":2499551,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T02:29:37.871145347Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2499551-2500408-2757815951/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:37.871202129Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2499551}
+{"time":"2026-07-23T02:29:38.054483267Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.064116376Z","level":"INFO","msg":"handleInformInit: received","streamId":"cemf0rge","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.350897746Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351103797Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351133399Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:29:38.351150646Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351193796Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351238698Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2499418-2500321-1544228546/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:38.352704212Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.352724918Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.352730597Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:29:38.445119298Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"cemf0rge","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796444376Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796598651Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796618722Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:29:38.796624128Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796668926Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796751175Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2499551-2500408-2757815951/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:38.797992882Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.798016256Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.798022072Z","level":"INFO","msg":"server is closed"}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_022937-cemf0rge/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-07-23T02:29:38.064274108Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-23T02:29:38.06450816Z","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-23T02:29:38.44495026Z","level":"INFO","msg":"stream: created new stream","id":"cemf0rge"}
+{"time":"2026-07-23T02:29:38.445115186Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-23T02:29:38.44526429Z","level":"INFO","msg":"writer: started","stream_id":"cemf0rge"}
+{"time":"2026-07-23T02:29:38.445290711Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-23T02:29:38.445315115Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-23T02:29:38.786524565Z","level":"ERROR","msg":"runupserter: failed to init run","error":"no data but must resume"}
+{"time":"2026-07-23T02:29:38.796867757Z","level":"ERROR","msg":"sender: uploadConfigFile: runhandle: run not yet initialized"}
+{"time":"2026-07-23T02:29:38.797204677Z","level":"ERROR","msg":"runfiles: runhandle: run not yet initialized"}
+{"time":"2026-07-23T02:29:38.797284772Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-07-23T02:29:38.797292885Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-07-23T02:29:38.797301918Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-07-23T02:29:38.797325603Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-07-23T02:29:38.797328171Z","level":"INFO","msg":"stream: all finished"}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_022937-cemf0rge/logs/debug.log

@@ -0,0 +1,13 @@
+2026-07-23 02:29:37,836 INFO    MainThread:2499551 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 02:29:37,836 INFO    MainThread:2499551 [wandb_setup.py:_flush():81] Configure stats pid to 2499551
+2026-07-23 02:29:37,836 INFO    MainThread:2499551 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 02:29:37,836 INFO    MainThread:2499551 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_022937-cemf0rge/logs/debug.log
+2026-07-23 02:29:37,836 INFO    MainThread:2499551 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_022937-cemf0rge/logs/debug-internal.log
+2026-07-23 02:29:37,836 INFO    MainThread:2499551 [wandb_init.py:init():766] calling init triggers
+2026-07-23 02:29:37,837 INFO    MainThread:2499551 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'cemf0rge', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}}
+2026-07-23 02:29:37,837 INFO    MainThread:2499551 [wandb_init.py:init():814] starting backend
+2026-07-23 02:29:38,054 INFO    MainThread:2499551 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 02:29:38,446 INFO    MainThread:2499551 [wandb_init.py:init():834] backend started and connected
+2026-07-23 02:29:38,447 INFO    MainThread:2499551 [wandb_init.py:init():904] updated telemetry
+2026-07-23 02:29:38,453 INFO    MainThread:2499551 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_023125-rs1egwv8/files/output.log

@@ -0,0 +1,149 @@
+INFO 2026-07-23 02:31:26 db_utils.py:126 Logs will be synced with wandb.
+INFO 2026-07-23 02:31:26 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/rs1egwv8
+INFO 2026-07-23 02:31:26 ot_train.py:307 Creating dataset
+INFO 2026-07-23 02:31:29 ot_train.py:341 Creating policy
+INFO 2026-07-23 02:31:29 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model
+INFO 2026-07-23 02:31:29 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 1256.77it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 807/807 [00:01<00:00, 580.05it/s] 
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                  | Status     |  |
+---------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.2.bias         | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.bias     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.2.weight       | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+Loading weights from local directory
+INFO 2026-07-23 02:31:45 ot_train.py:414 Creating optimizer and scheduler
+INFO 2026-07-23 02:31:47 ot_train.py:448 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+INFO 2026-07-23 02:31:47 ot_train.py:455 cfg.steps=60000 (60K)
+INFO 2026-07-23 02:31:47 ot_train.py:456 dataset.num_frames=218367 (218K)
+INFO 2026-07-23 02:31:47 ot_train.py:457 dataset.num_episodes=4930
+INFO 2026-07-23 02:31:47 ot_train.py:460 Effective batch size: 64 x 1 = 64
+INFO 2026-07-23 02:31:47 ot_train.py:461 num_learnable_params=1080755329 (1B)
+INFO 2026-07-23 02:31:47 ot_train.py:462 num_total_params=2604374177 (3B)
+INFO 2026-07-23 02:31:47 ot_train.py:503 Resuming data order at epoch 13, sample 42176
+Training:   0%|          | 0/30000 [00:00<?, ?step/s]INFO 2026-07-23 02:31:48 ot_train.py:613 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   1%|          | 200/30000 [05:04<11:42:04,  1.41s/step]INFO 2026-07-23 02:36:52 ot_train.py:658 step:30K smpl:2M ep:44K epch:8.85 loss:0.041 grdn:0.319 lr:5.4e-05 updt_s:0.810 data_s:0.712 smp/s:42 mem_gb:29.84
+Training:   1%|▏         | 400/30000 [09:58<11:50:43,  1.44s/step]INFO 2026-07-23 02:41:46 ot_train.py:658 step:30K smpl:2M ep:44K epch:8.91 loss:0.040 grdn:0.308 lr:5.3e-05 updt_s:0.777 data_s:0.689 smp/s:44 mem_gb:29.84
+Training:   2%|▏         | 600/30000 [14:54<11:47:28,  1.44s/step]INFO 2026-07-23 02:46:42 ot_train.py:658 step:31K smpl:2M ep:44K epch:8.97 loss:0.040 grdn:0.310 lr:5.3e-05 updt_s:0.775 data_s:0.698 smp/s:43 mem_gb:29.84
+Training:   3%|▎         | 800/30000 [19:52<12:08:23,  1.50s/step]INFO 2026-07-23 02:51:40 ot_train.py:658 step:31K smpl:2M ep:45K epch:9.03 loss:0.040 grdn:0.301 lr:5.2e-05 updt_s:0.772 data_s:0.716 smp/s:43 mem_gb:29.84
+Training:   3%|▎         | 1000/30000 [26:19<28:06:38,  3.49s/step]INFO 2026-07-23 02:58:07 ot_train.py:658 step:31K smpl:2M ep:45K epch:9.09 loss:0.039 grdn:0.292 lr:5.2e-05 updt_s:0.762 data_s:1.168 smp/s:33 mem_gb:29.84
+Training:   4%|▍         | 1200/30000 [34:20<17:28:09,  2.18s/step]INFO 2026-07-23 03:06:08 ot_train.py:658 step:31K smpl:2M ep:45K epch:9.14 loss:0.040 grdn:0.304 lr:5.1e-05 updt_s:0.786 data_s:1.620 smp/s:27 mem_gb:29.84
+Training:   5%|▍         | 1400/30000 [40:26<11:57:15,  1.50s/step]INFO 2026-07-23 03:12:14 ot_train.py:658 step:31K smpl:2M ep:45K epch:9.20 loss:0.040 grdn:0.336 lr:5.1e-05 updt_s:0.776 data_s:1.048 smp/s:35 mem_gb:29.84
+Training:   5%|▌         | 1600/30000 [46:11<27:09:05,  3.44s/step]INFO 2026-07-23 03:17:59 ot_train.py:658 step:32K smpl:2M ep:46K epch:9.26 loss:0.039 grdn:0.302 lr:5.0e-05 updt_s:0.825 data_s:0.896 smp/s:37 mem_gb:29.82
+Training:   6%|▌         | 1800/30000 [54:38<24:33:23,  3.13s/step]INFO 2026-07-23 03:26:26 ot_train.py:658 step:32K smpl:2M ep:46K epch:9.32 loss:0.037 grdn:0.286 lr:4.9e-05 updt_s:0.823 data_s:1.707 smp/s:25 mem_gb:29.85
+Training:   7%|▋         | 2000/30000 [1:01:43<14:06:08,  1.81s/step]INFO 2026-07-23 03:33:31 ot_train.py:658 step:32K smpl:2M ep:46K epch:9.38 loss:0.037 grdn:0.297 lr:4.9e-05 updt_s:0.807 data_s:1.319 smp/s:30 mem_gb:29.85
+Training:   7%|▋         | 2200/30000 [1:08:07<13:17:12,  1.72s/step]INFO 2026-07-23 03:39:55 ot_train.py:658 step:32K smpl:2M ep:47K epch:9.44 loss:0.037 grdn:0.289 lr:4.8e-05 updt_s:0.828 data_s:1.087 smp/s:33 mem_gb:29.85
+Training:   8%|▊         | 2400/30000 [1:13:30<9:55:12,  1.29s/step]INFO 2026-07-23 03:45:18 ot_train.py:658 step:32K smpl:2M ep:47K epch:9.50 loss:0.037 grdn:0.298 lr:4.8e-05 updt_s:0.809 data_s:0.801 smp/s:40 mem_gb:29.85
+Training:   9%|▊         | 2600/30000 [1:18:14<9:50:52,  1.29s/step]INFO 2026-07-23 03:50:02 ot_train.py:658 step:33K smpl:2M ep:47K epch:9.55 loss:0.037 grdn:0.286 lr:4.7e-05 updt_s:0.862 data_s:0.553 smp/s:45 mem_gb:29.85
+Training:   9%|▉         | 2800/30000 [1:22:35<9:47:49,  1.30s/step]INFO 2026-07-23 03:54:23 ot_train.py:658 step:33K smpl:2M ep:47K epch:9.61 loss:0.036 grdn:0.301 lr:4.7e-05 updt_s:0.765 data_s:0.539 smp/s:49 mem_gb:29.84
+Training:  10%|█         | 3000/30000 [1:27:06<9:32:39,  1.27s/step]INFO 2026-07-23 03:58:54 ot_train.py:658 step:33K smpl:2M ep:48K epch:9.67 loss:0.036 grdn:0.282 lr:4.6e-05 updt_s:0.772 data_s:0.578 smp/s:47 mem_gb:29.85
+Training:  11%|█         | 3200/30000 [1:32:14<33:05:48,  4.45s/step]INFO 2026-07-23 04:04:02 ot_train.py:658 step:33K smpl:2M ep:48K epch:9.73 loss:0.036 grdn:0.287 lr:4.6e-05 updt_s:0.888 data_s:0.648 smp/s:42 mem_gb:29.84
+Training:  11%|█▏        | 3400/30000 [1:38:58<10:00:45,  1.36s/step]INFO 2026-07-23 04:10:46 ot_train.py:658 step:33K smpl:2M ep:48K epch:9.79 loss:0.037 grdn:0.296 lr:4.5e-05 updt_s:1.109 data_s:0.908 smp/s:32 mem_gb:29.85
+Training:  12%|█▏        | 3600/30000 [1:43:42<9:21:23,  1.28s/step]INFO 2026-07-23 04:15:30 ot_train.py:658 step:34K smpl:2M ep:49K epch:9.85 loss:0.035 grdn:0.276 lr:4.4e-05 updt_s:0.799 data_s:0.618 smp/s:45 mem_gb:29.84
+Training:  13%|█▎        | 3800/30000 [1:48:01<9:15:45,  1.27s/step]INFO 2026-07-23 04:19:49 ot_train.py:658 step:34K smpl:2M ep:49K epch:9.91 loss:0.035 grdn:0.291 lr:4.4e-05 updt_s:0.745 data_s:0.546 smp/s:50 mem_gb:29.84
+Training:  13%|█▎        | 4000/30000 [1:52:21<9:11:59,  1.27s/step]INFO 2026-07-23 04:24:09 ot_train.py:658 step:34K smpl:2M ep:49K epch:9.96 loss:0.034 grdn:0.293 lr:4.3e-05 updt_s:0.744 data_s:0.554 smp/s:49 mem_gb:29.83
+Training:  14%|█▍        | 4200/30000 [1:56:41<9:46:57,  1.37s/step] INFO 2026-07-23 04:28:29 ot_train.py:658 step:34K smpl:2M ep:49K epch:10.02 loss:0.033 grdn:0.270 lr:4.3e-05 updt_s:0.748 data_s:0.550 smp/s:49 mem_gb:29.85
+Training:  15%|█▍        | 4400/30000 [2:00:59<9:23:54,  1.32s/step]INFO 2026-07-23 04:32:48 ot_train.py:658 step:34K smpl:2M ep:50K epch:10.08 loss:0.033 grdn:0.283 lr:4.2e-05 updt_s:0.739 data_s:0.549 smp/s:50 mem_gb:29.85
+Training:  15%|█▌        | 4600/30000 [2:05:24<9:16:51,  1.32s/step]INFO 2026-07-23 04:37:12 ot_train.py:658 step:35K smpl:2M ep:50K epch:10.14 loss:0.033 grdn:0.295 lr:4.2e-05 updt_s:0.753 data_s:0.568 smp/s:48 mem_gb:29.85
+Training:  16%|█▌        | 4800/30000 [2:09:46<9:00:30,  1.29s/step]INFO 2026-07-23 04:41:34 ot_train.py:658 step:35K smpl:2M ep:50K epch:10.20 loss:0.034 grdn:0.310 lr:4.1e-05 updt_s:0.742 data_s:0.563 smp/s:49 mem_gb:29.85
+Training:  17%|█▋        | 5000/30000 [2:14:08<9:19:05,  1.34s/step]INFO 2026-07-23 04:45:56 ot_train.py:658 step:35K smpl:2M ep:51K epch:10.26 loss:0.034 grdn:0.291 lr:4.1e-05 updt_s:0.730 data_s:0.578 smp/s:49 mem_gb:29.85
+INFO 2026-07-23 04:45:56 ot_train.py:703 Checkpoint policy after step 35000
+INFO 2026-07-23 04:46:20 in_yaml.py:1413 Saved verified rolling checkpoint step=35000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00
+Training:  17%|█▋        | 5200/30000 [2:18:55<8:50:43,  1.28s/step]INFO 2026-07-23 04:50:43 ot_train.py:658 step:35K smpl:2M ep:51K epch:10.32 loss:0.033 grdn:0.286 lr:4.0e-05 updt_s:0.774 data_s:0.539 smp/s:49 mem_gb:29.84
+Training:  18%|█▊        | 5400/30000 [2:23:14<8:57:44,  1.31s/step]INFO 2026-07-23 04:55:02 ot_train.py:658 step:35K smpl:2M ep:51K epch:10.38 loss:0.033 grdn:0.283 lr:4.0e-05 updt_s:0.775 data_s:0.514 smp/s:50 mem_gb:29.84
+Training:  19%|█▊        | 5600/30000 [2:27:37<8:55:10,  1.32s/step]INFO 2026-07-23 04:59:25 ot_train.py:658 step:36K smpl:2M ep:51K epch:10.43 loss:0.033 grdn:0.290 lr:3.9e-05 updt_s:0.784 data_s:0.530 smp/s:49 mem_gb:29.85
+Training:  19%|█▉        | 5800/30000 [2:32:04<8:54:23,  1.32s/step]INFO 2026-07-23 05:03:52 ot_train.py:658 step:36K smpl:2M ep:52K epch:10.49 loss:0.033 grdn:0.281 lr:3.9e-05 updt_s:0.765 data_s:0.569 smp/s:48 mem_gb:29.85
+Training:  20%|██        | 6000/30000 [2:37:07<10:46:18,  1.62s/step]INFO 2026-07-23 05:08:55 ot_train.py:658 step:36K smpl:2M ep:52K epch:10.55 loss:0.032 grdn:0.288 lr:3.8e-05 updt_s:0.824 data_s:0.687 smp/s:42 mem_gb:29.85
+Training:  21%|██        | 6200/30000 [2:42:34<10:40:44,  1.62s/step]INFO 2026-07-23 05:14:22 ot_train.py:658 step:36K smpl:2M ep:52K epch:10.61 loss:0.031 grdn:0.290 lr:3.7e-05 updt_s:0.841 data_s:0.787 smp/s:39 mem_gb:29.83
+Training:  21%|██▏       | 6400/30000 [2:48:24<10:48:23,  1.65s/step]INFO 2026-07-23 05:20:12 ot_train.py:658 step:36K smpl:2M ep:53K epch:10.67 loss:0.031 grdn:0.277 lr:3.7e-05 updt_s:0.876 data_s:0.868 smp/s:37 mem_gb:29.85
+Training:  22%|██▏       | 6600/30000 [2:53:49<10:25:44,  1.60s/step]INFO 2026-07-23 05:25:37 ot_train.py:658 step:37K smpl:2M ep:53K epch:10.73 loss:0.030 grdn:0.260 lr:3.6e-05 updt_s:0.846 data_s:0.777 smp/s:39 mem_gb:29.85
+Training:  23%|██▎       | 6800/30000 [2:59:31<10:51:40,  1.69s/step]INFO 2026-07-23 05:31:19 ot_train.py:658 step:37K smpl:2M ep:53K epch:10.79 loss:0.030 grdn:0.275 lr:3.6e-05 updt_s:0.862 data_s:0.845 smp/s:38 mem_gb:29.85
+Training:  23%|██▎       | 7000/30000 [3:04:57<10:34:46,  1.66s/step]INFO 2026-07-23 05:36:45 ot_train.py:658 step:37K smpl:2M ep:53K epch:10.84 loss:0.030 grdn:0.276 lr:3.5e-05 updt_s:0.834 data_s:0.792 smp/s:39 mem_gb:29.85
+Training:  24%|██▍       | 7200/30000 [3:10:20<10:05:08,  1.59s/step]INFO 2026-07-23 05:42:08 ot_train.py:658 step:37K smpl:2M ep:54K epch:10.90 loss:0.030 grdn:0.268 lr:3.5e-05 updt_s:0.836 data_s:0.771 smp/s:40 mem_gb:29.85
+Training:  25%|██▍       | 7400/30000 [3:15:40<10:13:24,  1.63s/step]INFO 2026-07-23 05:47:28 ot_train.py:658 step:37K smpl:2M ep:54K epch:10.96 loss:0.030 grdn:0.269 lr:3.4e-05 updt_s:0.834 data_s:0.764 smp/s:40 mem_gb:29.85
+Training:  25%|██▌       | 7600/30000 [3:20:55<9:42:54,  1.56s/step]INFO 2026-07-23 05:52:43 ot_train.py:658 step:38K smpl:2M ep:54K epch:11.02 loss:0.030 grdn:0.268 lr:3.4e-05 updt_s:0.816 data_s:0.758 smp/s:41 mem_gb:29.84
+Training:  26%|██▌       | 7800/30000 [3:26:13<9:42:03,  1.57s/step]INFO 2026-07-23 05:58:01 ot_train.py:658 step:38K smpl:2M ep:55K epch:11.08 loss:0.030 grdn:0.291 lr:3.3e-05 updt_s:0.805 data_s:0.777 smp/s:40 mem_gb:29.85
+Training:  27%|██▋       | 8000/30000 [3:31:27<10:16:26,  1.68s/step]INFO 2026-07-23 06:03:15 ot_train.py:658 step:38K smpl:2M ep:55K epch:11.14 loss:0.030 grdn:0.284 lr:3.3e-05 updt_s:0.789 data_s:0.780 smp/s:41 mem_gb:29.84
+Training:  27%|██▋       | 8200/30000 [3:36:42<9:23:31,  1.55s/step]INFO 2026-07-23 06:08:30 ot_train.py:658 step:38K smpl:2M ep:55K epch:11.20 loss:0.030 grdn:0.284 lr:3.2e-05 updt_s:0.840 data_s:0.729 smp/s:41 mem_gb:29.84
+Training:  28%|██▊       | 8400/30000 [3:41:56<8:53:52,  1.48s/step]INFO 2026-07-23 06:13:44 ot_train.py:658 step:38K smpl:2M ep:55K epch:11.25 loss:0.029 grdn:0.282 lr:3.2e-05 updt_s:0.862 data_s:0.706 smp/s:41 mem_gb:29.83
+Training:  29%|██▊       | 8600/30000 [3:47:16<9:34:08,  1.61s/step]INFO 2026-07-23 06:19:04 ot_train.py:658 step:39K smpl:2M ep:56K epch:11.31 loss:0.028 grdn:0.259 lr:3.1e-05 updt_s:0.822 data_s:0.775 smp/s:40 mem_gb:29.84
+Training:  29%|██▉       | 8800/30000 [3:53:46<27:38:09,  4.69s/step]INFO 2026-07-23 06:25:34 ot_train.py:658 step:39K smpl:2M ep:56K epch:11.37 loss:0.027 grdn:0.265 lr:3.1e-05 updt_s:0.770 data_s:1.176 smp/s:33 mem_gb:29.85
+Training:  30%|███       | 9000/30000 [4:02:29<13:54:12,  2.38s/step]INFO 2026-07-23 06:34:17 ot_train.py:658 step:39K smpl:2M ep:56K epch:11.43 loss:0.027 grdn:0.269 lr:3.0e-05 updt_s:0.786 data_s:1.826 smp/s:24 mem_gb:29.85
+Training:  31%|███       | 9200/30000 [4:09:19<10:05:13,  1.75s/step]INFO 2026-07-23 06:41:07 ot_train.py:658 step:39K smpl:3M ep:57K epch:11.49 loss:0.027 grdn:0.259 lr:3.0e-05 updt_s:0.772 data_s:1.273 smp/s:31 mem_gb:29.85
+Training:  31%|███▏      | 9400/30000 [4:17:43<14:44:45,  2.58s/step]INFO 2026-07-23 06:49:31 ot_train.py:658 step:39K smpl:3M ep:57K epch:11.55 loss:0.027 grdn:0.261 lr:2.9e-05 updt_s:0.739 data_s:1.778 smp/s:25 mem_gb:29.85
+Training:  32%|███▏      | 9600/30000 [4:23:55<9:45:19,  1.72s/step]INFO 2026-07-23 06:55:43 ot_train.py:658 step:40K smpl:3M ep:57K epch:11.61 loss:0.026 grdn:0.270 lr:2.9e-05 updt_s:0.854 data_s:1.001 smp/s:35 mem_gb:29.84
+Training:  33%|███▎      | 9800/30000 [4:29:32<9:19:33,  1.66s/step]INFO 2026-07-23 07:01:20 ot_train.py:658 step:40K smpl:3M ep:58K epch:11.66 loss:0.027 grdn:0.270 lr:2.8e-05 updt_s:0.820 data_s:0.858 smp/s:38 mem_gb:29.84
+Training:  33%|███▎      | 10000/30000 [4:35:41<9:49:56,  1.77s/step]INFO 2026-07-23 07:07:29 ot_train.py:658 step:40K smpl:3M ep:58K epch:11.72 loss:0.026 grdn:0.253 lr:2.8e-05 updt_s:0.834 data_s:1.007 smp/s:35 mem_gb:29.85
+INFO 2026-07-23 07:07:29 ot_train.py:703 Checkpoint policy after step 40000
+INFO 2026-07-23 07:07:58 in_yaml.py:1413 Saved verified rolling checkpoint step=40000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01
+Training:  34%|███▍      | 10200/30000 [4:42:42<8:51:50,  1.61s/step]INFO 2026-07-23 07:14:31 ot_train.py:658 step:40K smpl:3M ep:58K epch:11.78 loss:0.027 grdn:0.264 lr:2.7e-05 updt_s:0.815 data_s:1.142 smp/s:33 mem_gb:29.84
+Training:  35%|███▍      | 10400/30000 [4:48:12<9:06:45,  1.67s/step]INFO 2026-07-23 07:20:00 ot_train.py:658 step:40K smpl:3M ep:58K epch:11.84 loss:0.026 grdn:0.257 lr:2.7e-05 updt_s:0.838 data_s:0.805 smp/s:39 mem_gb:29.84
+Training:  35%|███▌      | 10600/30000 [4:53:30<8:13:24,  1.53s/step]INFO 2026-07-23 07:25:18 ot_train.py:658 step:41K smpl:3M ep:59K epch:11.90 loss:0.027 grdn:0.266 lr:2.6e-05 updt_s:0.792 data_s:0.792 smp/s:40 mem_gb:29.85
+Training:  36%|███▌      | 10800/30000 [4:58:55<9:23:44,  1.76s/step]INFO 2026-07-23 07:30:43 ot_train.py:658 step:41K smpl:3M ep:59K epch:11.96 loss:0.024 grdn:0.242 lr:2.6e-05 updt_s:0.810 data_s:0.815 smp/s:39 mem_gb:29.83
+Training:  37%|███▋      | 11000/30000 [5:04:48<9:02:44,  1.71s/step]INFO 2026-07-23 07:36:36 ot_train.py:658 step:41K smpl:3M ep:59K epch:12.02 loss:0.025 grdn:0.246 lr:2.5e-05 updt_s:0.870 data_s:0.890 smp/s:36 mem_gb:29.85
+Training:  37%|███▋      | 11200/30000 [5:10:37<8:47:55,  1.68s/step]INFO 2026-07-23 07:42:25 ot_train.py:658 step:41K smpl:3M ep:60K epch:12.08 loss:0.025 grdn:0.249 lr:2.5e-05 updt_s:0.850 data_s:0.891 smp/s:37 mem_gb:29.84
+Training:  38%|███▊      | 11400/30000 [5:16:48<18:00:35,  3.49s/step]INFO 2026-07-23 07:48:36 ot_train.py:658 step:41K smpl:3M ep:60K epch:12.13 loss:0.024 grdn:0.246 lr:2.4e-05 updt_s:0.818 data_s:1.029 smp/s:35 mem_gb:29.85
+Training:  39%|███▊      | 11600/30000 [5:24:19<11:00:04,  2.15s/step]INFO 2026-07-23 07:56:07 ot_train.py:658 step:42K smpl:3M ep:60K epch:12.19 loss:0.024 grdn:0.258 lr:2.4e-05 updt_s:0.789 data_s:1.466 smp/s:28 mem_gb:29.85
+Training:  39%|███▉      | 11800/30000 [5:30:49<16:03:56,  3.18s/step]INFO 2026-07-23 08:02:37 ot_train.py:658 step:42K smpl:3M ep:60K epch:12.25 loss:0.024 grdn:0.246 lr:2.3e-05 updt_s:0.792 data_s:1.154 smp/s:33 mem_gb:29.85
+Training:  40%|████      | 12000/30000 [5:37:13<8:10:00,  1.63s/step]INFO 2026-07-23 08:09:01 ot_train.py:658 step:42K smpl:3M ep:61K epch:12.31 loss:0.024 grdn:0.243 lr:2.3e-05 updt_s:0.796 data_s:1.117 smp/s:33 mem_gb:29.85
+Training:  41%|████      | 12200/30000 [5:43:13<14:44:41,  2.98s/step]INFO 2026-07-23 08:15:01 ot_train.py:658 step:42K smpl:3M ep:61K epch:12.37 loss:0.023 grdn:0.255 lr:2.2e-05 updt_s:0.802 data_s:0.994 smp/s:36 mem_gb:29.85
+Training:  41%|████▏     | 12400/30000 [5:48:53<9:33:34,  1.96s/step] INFO 2026-07-23 08:20:41 ot_train.py:658 step:42K smpl:3M ep:61K epch:12.43 loss:0.024 grdn:0.250 lr:2.2e-05 updt_s:0.832 data_s:0.865 smp/s:38 mem_gb:29.84
+Training:  42%|████▏     | 12600/30000 [5:56:03<13:31:05,  2.80s/step]INFO 2026-07-23 08:27:51 ot_train.py:658 step:43K smpl:3M ep:62K epch:12.49 loss:0.025 grdn:0.248 lr:2.2e-05 updt_s:0.807 data_s:1.341 smp/s:30 mem_gb:29.84
+Training:  43%|████▎     | 12800/30000 [6:06:50<60:52:50, 12.74s/step]INFO 2026-07-23 08:38:38 ot_train.py:658 step:43K smpl:3M ep:62K epch:12.54 loss:0.024 grdn:0.252 lr:2.1e-05 updt_s:0.798 data_s:2.432 smp/s:20 mem_gb:29.84
+Training:  43%|████▎     | 13000/30000 [6:14:13<7:49:04,  1.66s/step]INFO 2026-07-23 08:46:01 ot_train.py:658 step:43K smpl:3M ep:62K epch:12.60 loss:0.023 grdn:0.256 lr:2.1e-05 updt_s:0.821 data_s:1.391 smp/s:29 mem_gb:29.83
+Training:  44%|████▍     | 13200/30000 [6:19:43<7:34:42,  1.62s/step]INFO 2026-07-23 08:51:31 ot_train.py:658 step:43K smpl:3M ep:62K epch:12.66 loss:0.021 grdn:0.227 lr:2.0e-05 updt_s:0.812 data_s:0.834 smp/s:39 mem_gb:29.85
+Training:  45%|████▍     | 13400/30000 [6:25:08<7:51:44,  1.71s/step]INFO 2026-07-23 08:56:56 ot_train.py:658 step:43K smpl:3M ep:63K epch:12.72 loss:0.022 grdn:0.244 lr:2.0e-05 updt_s:0.804 data_s:0.816 smp/s:39 mem_gb:29.85
+Training:  45%|████▌     | 13600/30000 [6:30:32<7:26:31,  1.63s/step]INFO 2026-07-23 09:02:20 ot_train.py:658 step:44K smpl:3M ep:63K epch:12.78 loss:0.022 grdn:0.225 lr:1.9e-05 updt_s:0.802 data_s:0.813 smp/s:40 mem_gb:29.85
+Training:  46%|████▌     | 13800/30000 [6:36:38<7:49:33,  1.74s/step]INFO 2026-07-23 09:08:26 ot_train.py:658 step:44K smpl:3M ep:63K epch:12.84 loss:0.022 grdn:0.260 lr:1.9e-05 updt_s:0.789 data_s:1.036 smp/s:35 mem_gb:29.84
+Training:  47%|████▋     | 14000/30000 [6:42:39<7:57:32,  1.79s/step]INFO 2026-07-23 09:14:27 ot_train.py:658 step:44K smpl:3M ep:64K epch:12.90 loss:0.022 grdn:0.231 lr:1.8e-05 updt_s:0.811 data_s:0.993 smp/s:35 mem_gb:29.84
+Training:  47%|████▋     | 14200/30000 [6:48:23<8:18:40,  1.89s/step]INFO 2026-07-23 09:20:11 ot_train.py:658 step:44K smpl:3M ep:64K epch:12.95 loss:0.022 grdn:0.246 lr:1.8e-05 updt_s:0.830 data_s:0.883 smp/s:37 mem_gb:29.84
+Training:  48%|████▊     | 14400/30000 [6:55:47<9:01:51,  2.08s/step]INFO 2026-07-23 09:27:35 ot_train.py:658 step:44K smpl:3M ep:64K epch:13.01 loss:0.021 grdn:0.230 lr:1.8e-05 updt_s:0.799 data_s:1.417 smp/s:29 mem_gb:29.85
+Training:  49%|████▊     | 14600/30000 [7:01:25<6:27:45,  1.51s/step]INFO 2026-07-23 09:33:13 ot_train.py:658 step:45K smpl:3M ep:64K epch:13.07 loss:0.021 grdn:0.236 lr:1.7e-05 updt_s:0.804 data_s:0.885 smp/s:38 mem_gb:29.85
+Training:  49%|████▉     | 14800/30000 [7:06:44<7:06:24,  1.68s/step]INFO 2026-07-23 09:38:32 ot_train.py:658 step:45K smpl:3M ep:65K epch:13.13 loss:0.021 grdn:0.246 lr:1.7e-05 updt_s:0.824 data_s:0.765 smp/s:40 mem_gb:29.85
+Training:  50%|█████     | 15000/30000 [7:12:11<6:49:49,  1.64s/step]INFO 2026-07-23 09:43:59 ot_train.py:658 step:45K smpl:3M ep:65K epch:13.19 loss:0.022 grdn:0.227 lr:1.6e-05 updt_s:0.829 data_s:0.802 smp/s:39 mem_gb:29.85
+INFO 2026-07-23 09:43:59 ot_train.py:703 Checkpoint policy after step 45000
+INFO 2026-07-23 09:44:25 in_yaml.py:1413 Saved verified rolling checkpoint step=45000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00
+Training:  51%|█████     | 15200/30000 [7:18:00<7:01:12,  1.71s/step]INFO 2026-07-23 09:49:48 ot_train.py:658 step:45K smpl:3M ep:65K epch:13.25 loss:0.021 grdn:0.233 lr:1.6e-05 updt_s:0.812 data_s:0.800 smp/s:40 mem_gb:29.83
+Training:  51%|█████▏    | 15400/30000 [7:23:32<6:44:54,  1.66s/step]INFO 2026-07-23 09:55:20 ot_train.py:658 step:45K smpl:3M ep:66K epch:13.31 loss:0.020 grdn:0.218 lr:1.6e-05 updt_s:0.859 data_s:0.793 smp/s:39 mem_gb:29.85
+Training:  52%|█████▏    | 15600/30000 [7:28:51<6:04:24,  1.52s/step]INFO 2026-07-23 10:00:39 ot_train.py:658 step:46K smpl:3M ep:66K epch:13.36 loss:0.020 grdn:0.226 lr:1.5e-05 updt_s:0.814 data_s:0.777 smp/s:40 mem_gb:29.85
+Training:  53%|█████▎    | 15800/30000 [7:34:16<6:18:25,  1.60s/step]INFO 2026-07-23 10:06:04 ot_train.py:658 step:46K smpl:3M ep:66K epch:13.42 loss:0.020 grdn:0.241 lr:1.5e-05 updt_s:0.816 data_s:0.806 smp/s:39 mem_gb:29.85
+Training:  53%|█████▎    | 16000/30000 [7:40:32<6:54:11,  1.78s/step]INFO 2026-07-23 10:12:20 ot_train.py:658 step:46K smpl:3M ep:66K epch:13.48 loss:0.020 grdn:0.223 lr:1.4e-05 updt_s:0.801 data_s:1.074 smp/s:34 mem_gb:29.84
+Training:  54%|█████▍    | 16200/30000 [7:47:00<6:49:38,  1.78s/step]INFO 2026-07-23 10:18:48 ot_train.py:658 step:46K smpl:3M ep:67K epch:13.54 loss:0.020 grdn:0.233 lr:1.4e-05 updt_s:0.857 data_s:1.081 smp/s:33 mem_gb:29.85
+Training:  55%|█████▍    | 16400/30000 [7:53:59<6:23:57,  1.69s/step]INFO 2026-07-23 10:25:47 ot_train.py:658 step:46K smpl:3M ep:67K epch:13.60 loss:0.020 grdn:0.228 lr:1.4e-05 updt_s:0.825 data_s:1.264 smp/s:31 mem_gb:29.85
+Training:  55%|█████▌    | 16600/30000 [8:01:35<6:11:47,  1.66s/step]INFO 2026-07-23 10:33:23 ot_train.py:658 step:47K smpl:3M ep:67K epch:13.66 loss:0.019 grdn:0.227 lr:1.3e-05 updt_s:0.838 data_s:1.439 smp/s:28 mem_gb:29.85
+Training:  56%|█████▌    | 16800/30000 [8:07:39<5:53:47,  1.61s/step]INFO 2026-07-23 10:39:27 ot_train.py:658 step:47K smpl:3M ep:68K epch:13.72 loss:0.020 grdn:0.228 lr:1.3e-05 updt_s:0.856 data_s:0.960 smp/s:35 mem_gb:29.84
+Training:  57%|█████▋    | 17000/30000 [8:12:56<6:00:32,  1.66s/step]INFO 2026-07-23 10:44:44 ot_train.py:658 step:47K smpl:3M ep:68K epch:13.77 loss:0.019 grdn:0.228 lr:1.2e-05 updt_s:0.889 data_s:0.692 smp/s:40 mem_gb:29.84
+Training:  57%|█████▋    | 17200/30000 [8:27:00<17:27:42,  4.91s/step]INFO 2026-07-23 10:58:48 ot_train.py:658 step:47K smpl:3M ep:68K epch:13.83 loss:0.020 grdn:0.228 lr:1.2e-05 updt_s:3.367 data_s:0.850 smp/s:15 mem_gb:29.85
+Training:  58%|█████▊    | 17400/30000 [8:41:30<13:58:08,  3.99s/step]INFO 2026-07-23 11:13:18 ot_train.py:658 step:47K smpl:3M ep:68K epch:13.89 loss:0.019 grdn:0.225 lr:1.2e-05 updt_s:3.355 data_s:0.992 smp/s:15 mem_gb:29.83
+Training:  59%|█████▊    | 17600/30000 [8:55:45<18:19:28,  5.32s/step]INFO 2026-07-23 11:27:33 ot_train.py:658 step:48K smpl:3M ep:69K epch:13.95 loss:0.019 grdn:0.236 lr:1.1e-05 updt_s:3.422 data_s:0.849 smp/s:15 mem_gb:29.85
+Training:  59%|█████▉    | 17800/30000 [9:11:33<13:40:01,  4.03s/step]INFO 2026-07-23 11:43:21 ot_train.py:658 step:48K smpl:3M ep:69K epch:14.01 loss:0.018 grdn:0.225 lr:1.1e-05 updt_s:3.145 data_s:1.591 smp/s:14 mem_gb:29.85
+Training:  60%|██████    | 18000/30000 [9:25:29<13:42:28,  4.11s/step]INFO 2026-07-23 11:57:17 ot_train.py:658 step:48K smpl:3M ep:69K epch:14.07 loss:0.019 grdn:0.227 lr:1.1e-05 updt_s:3.320 data_s:0.854 smp/s:15 mem_gb:29.84
+Training:  61%|██████    | 18200/30000 [9:39:49<14:15:11,  4.35s/step]INFO 2026-07-23 12:11:37 ot_train.py:658 step:48K smpl:3M ep:70K epch:14.13 loss:0.018 grdn:0.230 lr:1.0e-05 updt_s:3.414 data_s:0.886 smp/s:15 mem_gb:29.85
+Training:  61%|██████▏   | 18400/30000 [9:53:56<14:28:13,  4.49s/step]INFO 2026-07-23 12:25:44 ot_train.py:658 step:48K smpl:3M ep:70K epch:14.19 loss:0.018 grdn:0.218 lr:1.0e-05 updt_s:3.465 data_s:0.766 smp/s:15 mem_gb:29.85
+Training:  62%|██████▏   | 18600/30000 [10:07:35<13:01:01,  4.11s/step]INFO 2026-07-23 12:39:23 ot_train.py:658 step:49K smpl:3M ep:70K epch:14.24 loss:0.018 grdn:0.226 lr:9.7e-06 updt_s:3.424 data_s:0.669 smp/s:16 mem_gb:29.85
+Training:  63%|██████▎   | 18800/30000 [10:21:17<13:50:28,  4.45s/step]INFO 2026-07-23 12:53:05 ot_train.py:658 step:49K smpl:3M ep:71K epch:14.30 loss:0.018 grdn:0.217 lr:9.4e-06 updt_s:3.456 data_s:0.649 smp/s:16 mem_gb:29.84
+Training:  63%|██████▎   | 19000/30000 [10:35:21<13:52:13,  4.54s/step]INFO 2026-07-23 13:07:09 ot_train.py:658 step:49K smpl:3M ep:71K epch:14.36 loss:0.018 grdn:0.209 lr:9.1e-06 updt_s:3.570 data_s:0.644 smp/s:15 mem_gb:29.84
+Training:  64%|██████▍   | 19200/30000 [10:48:56<10:50:18,  3.61s/step]INFO 2026-07-23 13:20:44 ot_train.py:658 step:49K smpl:3M ep:71K epch:14.42 loss:0.018 grdn:0.219 lr:8.8e-06 updt_s:3.473 data_s:0.600 smp/s:16 mem_gb:29.84
+Training:  65%|██████▍   | 19400/30000 [11:03:04<12:04:34,  4.10s/step]INFO 2026-07-23 13:34:52 ot_train.py:658 step:49K smpl:3M ep:71K epch:14.48 loss:0.018 grdn:0.217 lr:8.4e-06 updt_s:3.473 data_s:0.765 smp/s:15 mem_gb:29.84
+Training:  65%|██████▌   | 19600/30000 [11:17:00<10:49:27,  3.75s/step]INFO 2026-07-23 13:48:48 ot_train.py:658 step:50K smpl:3M ep:72K epch:14.54 loss:0.017 grdn:0.226 lr:8.1e-06 updt_s:3.486 data_s:0.694 smp/s:15 mem_gb:29.85
+Training:  66%|██████▌   | 19800/30000 [11:30:48<12:12:01,  4.31s/step]INFO 2026-07-23 14:02:36 ot_train.py:658 step:50K smpl:3M ep:72K epch:14.60 loss:0.017 grdn:0.216 lr:7.8e-06 updt_s:3.490 data_s:0.643 smp/s:15 mem_gb:29.83
+Training:  67%|██████▋   | 20000/30000 [11:44:24<11:03:22,  3.98s/step]INFO 2026-07-23 14:16:12 ot_train.py:658 step:50K smpl:3M ep:72K epch:14.65 loss:0.017 grdn:0.210 lr:7.5e-06 updt_s:3.495 data_s:0.583 smp/s:16 mem_gb:29.84
+INFO 2026-07-23 14:16:12 ot_train.py:703 Checkpoint policy after step 50000
+INFO 2026-07-23 14:16:43 in_yaml.py:1413 Saved verified rolling checkpoint step=50000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01
+Training:  67%|██████▋   | 20082/30000 [11:50:29<10:53:43,  3.95s/step]

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_023125-rs1egwv8/files/requirements.txt

@@ -0,0 +1,117 @@
+tzdata==2026.3
+nvidia-cufile-cu12==1.13.1.3
+wandb==0.27.2
+six==1.17.0
+peft==0.19.1
+nvidia-curand-cu12==10.3.9.90
+nvidia-cuda-runtime-cu12==12.8.90
+datasets==4.8.5
+attrs==26.1.0
+cloudpickle==3.1.2
+packaging==25.0
+nvidia-cublas-cu12==12.8.4.1
+cmake==4.1.3
+psutil==7.2.2
+pydantic==2.13.4
+GitPython==3.1.51
+pillow==12.2.0
+pytz==2026.2
+cuda-pathfinder==1.2.2
+nvidia-cusparse-cu12==12.5.8.93
+typing_extensions==4.15.0
+urllib3==2.7.0
+pydantic_core==2.46.4
+draccus==0.10.0
+mypy_extensions==1.1.0
+dill==0.4.1
+typer==0.26.8
+decord==0.6.0
+tqdm==4.68.4
+triton==3.6.0
+idna==3.18
+requests==2.34.2
+nvidia-cudnn-cu12==9.19.0.56
+mdurl==0.1.2
+shellingham==1.5.4
+zipp==4.1.0
+timm==1.0.28
+torch==2.11.0+cu128
+tokenizers==0.22.2
+nvidia-cusparselt-cu12==0.7.1
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cusolver-cu12==11.7.3.90
+opencv-python-headless==4.13.0.92
+httpx==0.28.1
+Jinja2==3.1.6
+einops==0.8.2
+mergedeep==1.3.4
+platformdirs==4.10.0
+nvidia-nvjitlink-cu12==12.8.93
+lerobot==0.6.0
+PyYAML==6.0.3
+accelerate==1.14.0
+anyio==4.14.2
+gymnasium==1.3.0
+pip==26.1.2
+huggingface_hub==1.23.0
+diffusers==0.35.2
+annotated-doc==0.0.4
+pyarrow==25.0.0
+python-dateutil==2.9.0.post0
+termcolor==3.3.0
+nvidia-nccl-cu12==2.28.9
+sentry-sdk==2.65.0
+torchcodec==0.11.1
+pyyaml-include==1.4.1
+multiprocess==0.70.19
+wrapt==2.2.2
+typing-inspection==0.4.2
+nvidia-cuda-nvrtc-cu12==12.8.93
+Pygments==2.20.0
+nvidia-cuda-cupti-cu12==12.8.90
+certifi==2026.6.17
+setuptools==78.1.0
+charset-normalizer==3.4.9
+propcache==0.5.2
+multidict==6.7.1
+regex==2026.7.10
+frozenlist==1.8.0
+smmap==5.0.3
+hf-xet==1.5.1
+torchvision==0.26.0+cu128
+gitdb==4.0.12
+transformers==5.5.4
+rich==15.0.0
+cuda-bindings==12.9.4
+protobuf==7.35.1
+aiosignal==1.4.0
+jsonlines==4.0.0
+yarl==1.24.2
+filelock==3.29.0
+av==15.1.0
+networkx==3.6.1
+sympy==1.14.0
+wheel==0.47.0
+nvidia-nvtx-cu12==12.8.90
+xxhash==3.8.1
+toml==0.10.2
+MarkupSafe==3.0.3
+pandas==2.3.3
+safetensors==0.8.0
+click==8.4.2
+fsspec==2026.2.0
+dm-tree==0.1.10
+absl-py==2.5.0
+Farama-Notifications==0.0.6
+mpmath==1.3.0
+importlib_metadata==9.0.0
+markdown-it-py==4.2.0
+h11==0.16.0
+typing-inspect==0.9.0
+httpcore==1.0.9
+annotated-types==0.7.0
+numpy==2.2.6
+nvidia-nvshmem-cu12==3.4.5
+aiohappyeyeballs==2.7.1
+cuda-toolkit==12.8.1
+aiohttp==3.14.1

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_023125-rs1egwv8/files/wandb-metadata.json

@@ -0,0 +1,53 @@
+{
+  "os": "Linux-6.8.0-55-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-23T02:31:25.059506Z",
+  "args": [
+    "--config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/train_config.json",
+    "--resume=true",
+    "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K",
+    "--dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+    "--wandb.run_id=rs1egwv8"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePathLocal": "my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a329728c2c9524d79b966c4a7a82ee56a4c2abf7"
+  },
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K",
+  "host": "ext_minje-box",
+  "executable": "/home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 2,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "1138103234560"
+    }
+  },
+  "memory": {
+    "total": "2164176568320"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "rcam6hzew8b6nlqutd1m9r4avsa0csph"
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_023125-rs1egwv8/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2026-07-23T02:31:25.095933268Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphmwi8v6n/port-2501080.txt","pid":2501080,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T02:31:25.096932569Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2501080}
+{"time":"2026-07-23T02:31:25.096927661Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2501080-2501865-1162647384/socket","Net":"unix"}}
+{"time":"2026-07-23T02:31:25.169461847Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4t2rwfks/port-2500816.txt","pid":2500816,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T02:31:25.170134264Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2500816-2501888-288489369/socket","Net":"unix"}}
+{"time":"2026-07-23T02:31:25.170259549Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2500816}
+{"time":"2026-07-23T02:31:25.27857788Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T02:31:25.285691576Z","level":"INFO","msg":"handleInformInit: received","streamId":"rs1egwv8","id":"1(@)"}
+{"time":"2026-07-23T02:31:25.348726979Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T02:31:25.356108831Z","level":"INFO","msg":"handleInformInit: received","streamId":"x6da8gup","id":"1(@)"}
+{"time":"2026-07-23T02:31:25.661339067Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rs1egwv8","id":"1(@)"}
+{"time":"2026-07-23T02:31:25.811527394Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"x6da8gup","id":"1(@)"}
+{"time":"2026-07-23T02:31:31.694097963Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"7skxzehhhgck"}
+{"time":"2026-07-23T02:31:31.708133393Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"b3q6giuvspld"}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_023125-rs1egwv8/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_setup.py:_flush():81] Configure stats pid to 2501080
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_023125-rs1egwv8/logs/debug.log
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_023125-rs1egwv8/logs/debug-internal.log
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_init.py:init():766] calling init triggers
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'rs1egwv8', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}}
+2026-07-23 02:31:25,060 INFO    MainThread:2501080 [wandb_init.py:init():814] starting backend
+2026-07-23 02:31:25,278 INFO    MainThread:2501080 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 02:31:25,662 INFO    MainThread:2501080 [wandb_init.py:init():834] backend started and connected
+2026-07-23 02:31:25,664 INFO    MainThread:2501080 [wandb_init.py:init():904] updated telemetry
+2026-07-23 02:31:25,668 INFO    MainThread:2501080 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 02:31:26,617 INFO    MainThread:2501080 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 02:31:26,676 INFO    MainThread:2501080 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 02:31:26,677 INFO    MainThread:2501080 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 02:31:26,677 INFO    MainThread:2501080 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 02:31:26,677 INFO    MainThread:2501080 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 02:31:26,678 INFO    MainThread:2501080 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/files/config.yaml

@@ -0,0 +1,329 @@
+_wandb:
+    value:
+        cli_version: 0.27.2
+        e:
+            3vx7sedbdsz3uay6fuuuh2e3qlqtbnr9:
+                args:
+                    - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/train_config.json
+                    - --resume=true
+                codePath: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                codePathLocal: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.9"
+                disk:
+                    /:
+                        total: "1888425144320"
+                        used: "1327086579712"
+                executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python
+                gpu: NVIDIA H200
+                gpu_count: 2
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc
+                host: ext_minje-box
+                memory:
+                    total: "2164176568320"
+                os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39
+                program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                python: CPython 3.12.13
+                root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+                startedAt: "2026-07-23T16:35:01.611693Z"
+                writerId: 3vx7sedbdsz3uay6fuuuh2e3qlqtbnr9
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.12.13
+            "5": 0.27.2
+            "6": 5.5.4
+            "10":
+                - 21
+            "12": 0.27.2
+            "13": linux-x86_64
+batch_size:
+    value: 64
+checkpoint_path:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01
+cudnn_deterministic:
+    value: false
+dataset:
+    value:
+        depth_output_unit: mm
+        episodes: null
+        eval_split: 0
+        image_transforms:
+            enable: false
+            max_num_transforms: 3
+            random_order: false
+            tfs:
+                affine:
+                    kwargs:
+                        degrees:
+                            - -5
+                            - 5
+                        translate:
+                            - 0.05
+                            - 0.05
+                    type: RandomAffine
+                    weight: 1
+                brightness:
+                    kwargs:
+                        brightness:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                contrast:
+                    kwargs:
+                        contrast:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                hue:
+                    kwargs:
+                        hue:
+                            - -0.05
+                            - 0.05
+                    type: ColorJitter
+                    weight: 1
+                saturation:
+                    kwargs:
+                        saturation:
+                            - 0.5
+                            - 1.5
+                    type: ColorJitter
+                    weight: 1
+                sharpness:
+                    kwargs:
+                        sharpness:
+                            - 0.5
+                            - 1.5
+                    type: SharpnessJitter
+                    weight: 1
+        repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+        return_uint8: false
+        revision: null
+        root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+        streaming: false
+        use_imagenet_stats: true
+        video_backend: torchcodec
+env:
+    value: null
+env_eval_freq:
+    value: 20000
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        recording: false
+        recording_private: false
+        recording_repo_id: null
+        use_async_envs: true
+eval_steps:
+    value: 0
+job:
+    value:
+        detach: false
+        image: huggingface/lerobot-gpu:latest
+        tags: []
+        target: null
+        timeout: 2d
+job_name:
+    value: INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60
+log_freq:
+    value: 200
+max_eval_samples:
+    value: 0
+num_workers:
+    value: 4
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.999
+        eps: 1e-08
+        grad_clip_norm: 1
+        lr: 0.0001
+        type: adamw
+        weight_decay: 1e-05
+output_dir:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+peft:
+    value: null
+persistent_workers:
+    value: true
+policy:
+    value:
+        action_decode_transform: null
+        action_expert_num_layers: 16
+        balance_dataset_weights: true
+        balance_trajectory_weights: true
+        base_model_path: nvidia/GR00T-N1.7-3B
+        batch_size: 32
+        chunk_size: 16
+        dataloader_num_workers: 8
+        dataset_paths: null
+        device: cuda
+        embodiment_tag: new_embodiment
+        image_size:
+            - 256
+            - 256
+        input_features:
+            observation.images.guide:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.right_shoulder:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.wrist:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.state:
+                shape:
+                    - 16
+                type: STATE
+        license: null
+        lora_alpha: 16
+        lora_dropout: 0.1
+        lora_full_model: false
+        lora_rank: 0
+        max_action_dim: 132
+        max_state_dim: 132
+        max_steps: 10000
+        model_params_fp32: true
+        n_action_steps: 16
+        n_obs_steps: 1
+        normalization_mapping:
+            ACTION: IDENTITY
+            STATE: IDENTITY
+            VISUAL: IDENTITY
+        num_inference_timesteps: null
+        optimizer_betas:
+            - 0.9
+            - 0.999
+        optimizer_eps: 1e-08
+        optimizer_lr: 0.0001
+        optimizer_weight_decay: 1e-05
+        output_dir: ./tmp/gr00t
+        output_features:
+            action:
+                shape:
+                    - 10
+                type: ACTION
+        pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model
+        pretrained_revision: null
+        private: null
+        push_to_hub: false
+        relative_exclude_joints: []
+        repo_id: null
+        report_to: wandb
+        resume: false
+        rkd_angle_weight: 2
+        rkd_distance_weight: 1
+        rkd_enabled: true
+        rkd_fm_loss_weight: 1
+        rkd_loss_weight: 1
+        rkd_student_attention_hidden_dim: 512
+        rkd_student_attention_output_dim: 512
+        rkd_student_pooling: attention
+        rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11
+        rtc_ramp_rate: null
+        save_steps: 1000
+        tags: null
+        tokenizer_assets_repo: null
+        tune_diffusion_model: true
+        tune_llm: false
+        tune_projector: true
+        tune_top_llm_layers: 0
+        tune_visual: false
+        tune_vlln: true
+        type: groot_rkd
+        use_amp: false
+        use_bf16: true
+        use_flash_attention: false
+        use_peft: false
+        use_relative_actions: false
+        video_backend: decord
+        warmup_ratio: 0.05
+        warmup_steps: 3000
+prefetch_factor:
+    value: 4
+resume:
+    value: true
+reward_model:
+    value: null
+sample_weighting:
+    value: null
+save_checkpoint:
+    value: true
+save_checkpoint_to_hub:
+    value: false
+save_freq:
+    value: 20000
+scheduler:
+    value:
+        name: cosine
+        num_warmup_steps: 3000
+        type: diffuser
+seed:
+    value: 42
+steps:
+    value: 60000
+tolerance_s:
+    value: 0.0001
+use_policy_training_preset:
+    value: true
+wandb:
+    value:
+        add_tags: true
+        disable_artifact: true
+        enable: true
+        entity: null
+        mode: null
+        notes: null
+        project: lerobot
+        run_id: rs1egwv8

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/files/output.log

@@ -0,0 +1,96 @@
+INFO 2026-07-23 16:35:03 db_utils.py:126 Logs will be synced with wandb.
+INFO 2026-07-23 16:35:03 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/rs1egwv8
+INFO 2026-07-23 16:35:03 ot_train.py:310 Creating dataset
+INFO 2026-07-23 16:35:05 ot_train.py:344 Creating policy
+INFO 2026-07-23 16:35:05 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model
+INFO 2026-07-23 16:35:05 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 5334.76it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 807/807 [00:04<00:00, 187.16it/s]
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                  | Status     |  |
+---------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.2.weight       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.2.bias         | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.bias     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+Loading weights from local directory
+INFO 2026-07-23 16:35:27 ot_train.py:417 Creating optimizer and scheduler
+INFO 2026-07-23 16:35:31 ot_train.py:451 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K
+INFO 2026-07-23 16:35:31 ot_train.py:458 cfg.steps=60000 (60K)
+INFO 2026-07-23 16:35:31 ot_train.py:459 dataset.num_frames=218367 (218K)
+INFO 2026-07-23 16:35:31 ot_train.py:460 dataset.num_episodes=4930
+INFO 2026-07-23 16:35:31 ot_train.py:463 Effective batch size: 64 x 1 = 64
+INFO 2026-07-23 16:35:31 ot_train.py:464 num_learnable_params=1080755329 (1B)
+INFO 2026-07-23 16:35:31 ot_train.py:465 num_total_params=2604374177 (3B)
+INFO 2026-07-23 16:35:31 ot_train.py:506 Resuming data order at epoch 22, sample 22144
+Training:   0%|          | 0/10000 [00:00<?, ?step/s]INFO 2026-07-23 16:35:31 ot_train.py:616 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   2%|▏         | 200/10000 [13:50<10:59:57,  4.04s/step]INFO 2026-07-23 16:49:22 ot_train.py:661 step:50K smpl:3M ep:73K epch:14.71 loss:0.017 grdn:0.208 lr:7.3e-06 updt_s:3.597 data_s:0.554 smp/s:15 mem_gb:29.84 flow_matching_loss:0.017 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.017 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.066 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.389 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:   4%|▍         | 400/10000 [27:29<10:27:16,  3.92s/step]INFO 2026-07-23 17:03:00 ot_train.py:661 step:50K smpl:3M ep:73K epch:14.77 loss:0.017 grdn:0.211 lr:7.0e-06 updt_s:3.559 data_s:0.531 smp/s:16 mem_gb:29.84 flow_matching_loss:0.017 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.017 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.063 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.330 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:   6%|▌         | 600/10000 [41:05<10:16:11,  3.93s/step]INFO 2026-07-23 17:16:36 ot_train.py:661 step:51K smpl:3M ep:73K epch:14.83 loss:0.017 grdn:0.226 lr:6.7e-06 updt_s:3.547 data_s:0.530 smp/s:16 mem_gb:29.84 flow_matching_loss:0.017 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.017 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.061 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.348 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:   8%|▊         | 800/10000 [54:43<10:41:44,  4.19s/step]INFO 2026-07-23 17:30:15 ot_train.py:661 step:51K smpl:3M ep:73K epch:14.89 loss:0.016 grdn:0.214 lr:6.4e-06 updt_s:3.563 data_s:0.529 smp/s:16 mem_gb:29.84 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.058 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.331 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  10%|█         | 1000/10000 [1:08:20<10:21:49,  4.15s/step]INFO 2026-07-23 17:43:51 ot_train.py:661 step:51K smpl:3M ep:74K epch:14.95 loss:0.016 grdn:0.210 lr:6.2e-06 updt_s:3.548 data_s:0.530 smp/s:16 mem_gb:29.84 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.056 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.340 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  12%|█▏        | 1200/10000 [1:21:55<10:12:47,  4.18s/step]INFO 2026-07-23 17:57:27 ot_train.py:661 step:51K smpl:3M ep:74K epch:15.01 loss:0.016 grdn:0.210 lr:5.9e-06 updt_s:3.543 data_s:0.531 smp/s:16 mem_gb:29.84 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.053 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.291 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  14%|█▍        | 1400/10000 [1:35:34<10:08:33,  4.25s/step]INFO 2026-07-23 18:11:05 ot_train.py:661 step:51K smpl:3M ep:74K epch:15.06 loss:0.016 grdn:0.213 lr:5.6e-06 updt_s:3.560 data_s:0.530 smp/s:16 mem_gb:29.84 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.051 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.284 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  16%|█▌        | 1600/10000 [1:49:11<9:27:12,  4.05s/step]INFO 2026-07-23 18:24:42 ot_train.py:661 step:52K smpl:3M ep:75K epch:15.12 loss:0.016 grdn:0.212 lr:5.4e-06 updt_s:3.564 data_s:0.521 smp/s:16 mem_gb:29.84 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.049 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.236 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  18%|█▊        | 1800/10000 [2:02:49<8:58:00,  3.94s/step]INFO 2026-07-23 18:38:20 ot_train.py:661 step:52K smpl:3M ep:75K epch:15.18 loss:0.017 grdn:0.215 lr:5.1e-06 updt_s:3.563 data_s:0.525 smp/s:16 mem_gb:29.83 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.046 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.253 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  20%|██        | 2000/10000 [2:16:31<9:08:44,  4.12s/step]INFO 2026-07-23 18:52:03 ot_train.py:661 step:52K smpl:3M ep:75K epch:15.24 loss:0.016 grdn:0.220 lr:4.9e-06 updt_s:3.584 data_s:0.527 smp/s:16 mem_gb:29.82 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.044 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.249 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  22%|██▏       | 2200/10000 [2:30:16<9:17:43,  4.29s/step]INFO 2026-07-23 19:05:48 ot_train.py:661 step:52K smpl:3M ep:75K epch:15.30 loss:0.016 grdn:0.216 lr:4.7e-06 updt_s:3.590 data_s:0.531 smp/s:16 mem_gb:29.84 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.042 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.216 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  24%|██▍       | 2400/10000 [2:43:59<9:00:55,  4.27s/step]INFO 2026-07-23 19:19:31 ot_train.py:661 step:52K smpl:3M ep:76K epch:15.36 loss:0.016 grdn:0.205 lr:4.4e-06 updt_s:3.586 data_s:0.528 smp/s:16 mem_gb:29.85 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.040 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.197 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  26%|██▌       | 2600/10000 [2:57:34<8:36:33,  4.19s/step]INFO 2026-07-23 19:33:06 ot_train.py:661 step:53K smpl:3M ep:76K epch:15.42 loss:0.015 grdn:0.201 lr:4.2e-06 updt_s:3.546 data_s:0.527 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.038 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.223 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  28%|██▊       | 2800/10000 [3:11:19<8:27:43,  4.23s/step]INFO 2026-07-23 19:46:51 ot_train.py:661 step:53K smpl:3M ep:76K epch:15.47 loss:0.016 grdn:0.206 lr:4.0e-06 updt_s:3.593 data_s:0.530 smp/s:16 mem_gb:29.84 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.036 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.215 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  30%|███       | 3000/10000 [3:24:53<8:18:47,  4.28s/step]INFO 2026-07-23 20:00:24 ot_train.py:661 step:53K smpl:3M ep:77K epch:15.53 loss:0.016 grdn:0.211 lr:3.8e-06 updt_s:3.536 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.034 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.220 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  32%|███▏      | 3200/10000 [3:38:26<7:18:43,  3.87s/step]INFO 2026-07-23 20:13:57 ot_train.py:661 step:53K smpl:3M ep:77K epch:15.59 loss:0.016 grdn:0.219 lr:3.6e-06 updt_s:3.536 data_s:0.528 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.032 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.196 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  34%|███▍      | 3400/10000 [3:52:03<7:36:05,  4.15s/step]INFO 2026-07-23 20:27:35 ot_train.py:661 step:53K smpl:3M ep:77K epch:15.65 loss:0.016 grdn:0.205 lr:3.4e-06 updt_s:3.557 data_s:0.527 smp/s:16 mem_gb:29.85 flow_matching_loss:0.016 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.016 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.030 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.206 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  36%|███▌      | 3600/10000 [4:05:39<7:04:13,  3.98s/step]INFO 2026-07-23 20:41:11 ot_train.py:661 step:54K smpl:3M ep:77K epch:15.71 loss:0.015 grdn:0.208 lr:3.2e-06 updt_s:3.551 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.029 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.184 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  38%|███▊      | 3800/10000 [4:19:18<6:57:11,  4.04s/step]INFO 2026-07-23 20:54:50 ot_train.py:661 step:54K smpl:3M ep:78K epch:15.77 loss:0.015 grdn:0.198 lr:3.0e-06 updt_s:3.566 data_s:0.526 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.027 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.188 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  40%|████      | 4000/10000 [4:32:56<6:54:03,  4.14s/step]INFO 2026-07-23 21:08:28 ot_train.py:661 step:54K smpl:3M ep:78K epch:15.83 loss:0.016 grdn:0.206 lr:2.8e-06 updt_s:3.556 data_s:0.533 smp/s:16 mem_gb:29.84 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.025 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.214 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  42%|████▏     | 4200/10000 [4:46:35<6:24:18,  3.98s/step]INFO 2026-07-23 21:22:07 ot_train.py:661 step:54K smpl:3M ep:78K epch:15.89 loss:0.015 grdn:0.207 lr:2.6e-06 updt_s:3.561 data_s:0.531 smp/s:16 mem_gb:29.83 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.024 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.198 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  44%|████▍     | 4400/10000 [5:00:15<6:04:48,  3.91s/step]INFO 2026-07-23 21:35:47 ot_train.py:661 step:54K smpl:3M ep:79K epch:15.94 loss:0.015 grdn:0.204 lr:2.4e-06 updt_s:3.570 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.022 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.194 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  46%|████▌     | 4600/10000 [5:13:56<6:00:53,  4.01s/step]INFO 2026-07-23 21:49:28 ot_train.py:661 step:55K smpl:3M ep:79K epch:16.00 loss:0.015 grdn:0.206 lr:2.3e-06 updt_s:3.574 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.021 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.198 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  48%|████▊     | 4800/10000 [5:27:34<6:00:41,  4.16s/step]INFO 2026-07-23 22:03:06 ot_train.py:661 step:55K smpl:4M ep:79K epch:16.06 loss:0.015 grdn:0.207 lr:2.1e-06 updt_s:3.560 data_s:0.527 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.019 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.200 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  50%|█████     | 5000/10000 [5:41:10<4:26:09,  3.19s/step]INFO 2026-07-23 22:16:42 ot_train.py:661 step:55K smpl:4M ep:79K epch:16.12 loss:0.015 grdn:0.206 lr:2.0e-06 updt_s:3.550 data_s:0.530 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.018 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.203 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+INFO 2026-07-23 22:16:42 ot_train.py:707 Checkpoint policy after step 55000
+INFO 2026-07-23 22:17:00 in_yaml.py:1414 Saved verified rolling checkpoint step=55000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00
+Training:  52%|█████▏    | 5200/10000 [5:55:11<5:27:42,  4.10s/step]INFO 2026-07-23 22:30:43 ot_train.py:661 step:55K smpl:4M ep:80K epch:16.18 loss:0.015 grdn:0.200 lr:1.8e-06 updt_s:3.582 data_s:0.530 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.016 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.189 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  54%|█████▍    | 5400/10000 [6:08:50<5:03:22,  3.96s/step]INFO 2026-07-23 22:44:21 ot_train.py:661 step:55K smpl:4M ep:80K epch:16.24 loss:0.015 grdn:0.215 lr:1.7e-06 updt_s:3.561 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.015 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.212 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  56%|█████▌    | 5600/10000 [6:22:31<5:13:40,  4.28s/step]INFO 2026-07-23 22:58:02 ot_train.py:661 step:56K smpl:4M ep:80K epch:16.30 loss:0.015 grdn:0.206 lr:1.5e-06 updt_s:3.573 data_s:0.530 smp/s:16 mem_gb:29.84 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.014 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.192 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  58%|█████▊    | 5800/10000 [6:36:07<4:38:04,  3.97s/step]INFO 2026-07-23 23:11:39 ot_train.py:661 step:56K smpl:4M ep:81K epch:16.35 loss:0.015 grdn:0.205 lr:1.4e-06 updt_s:3.551 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.013 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.155 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  60%|██████    | 6000/10000 [6:49:37<4:33:48,  4.11s/step]INFO 2026-07-23 23:25:09 ot_train.py:661 step:56K smpl:4M ep:81K epch:16.41 loss:0.015 grdn:0.201 lr:1.3e-06 updt_s:3.520 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.011 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.122 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  62%|██████▏   | 6200/10000 [7:03:15<4:17:22,  4.06s/step]INFO 2026-07-23 23:38:47 ot_train.py:661 step:56K smpl:4M ep:81K epch:16.47 loss:0.015 grdn:0.201 lr:1.2e-06 updt_s:3.556 data_s:0.530 smp/s:16 mem_gb:29.85 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.010 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.126 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  64%|██████▍   | 6400/10000 [7:16:53<4:03:36,  4.06s/step]INFO 2026-07-23 23:52:24 ot_train.py:661 step:56K smpl:4M ep:81K epch:16.53 loss:0.015 grdn:0.199 lr:1.0e-06 updt_s:3.558 data_s:0.527 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.009 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.127 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  66%|██████▌   | 6600/10000 [7:30:32<3:53:04,  4.11s/step]INFO 2026-07-24 00:06:03 ot_train.py:661 step:57K smpl:4M ep:82K epch:16.59 loss:0.015 grdn:0.200 lr:9.3e-07 updt_s:3.565 data_s:0.529 smp/s:16 mem_gb:29.83 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.008 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.126 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  68%|██████▊   | 6800/10000 [7:44:09<3:46:36,  4.25s/step]INFO 2026-07-24 00:19:40 ot_train.py:661 step:57K smpl:4M ep:82K epch:16.65 loss:0.015 grdn:0.207 lr:8.2e-07 updt_s:3.551 data_s:0.531 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.007 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.137 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  70%|███████   | 7000/10000 [7:57:46<3:25:39,  4.11s/step]INFO 2026-07-24 00:33:17 ot_train.py:661 step:57K smpl:4M ep:82K epch:16.71 loss:0.015 grdn:0.207 lr:7.3e-07 updt_s:3.554 data_s:0.529 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.007 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.128 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  72%|███████▏  | 7200/10000 [8:11:20<3:15:16,  4.18s/step]INFO 2026-07-24 00:46:51 ot_train.py:661 step:57K smpl:4M ep:83K epch:16.76 loss:0.015 grdn:0.201 lr:6.4e-07 updt_s:3.535 data_s:0.532 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.006 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.146 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  74%|███████▍  | 7400/10000 [8:24:58<2:50:34,  3.94s/step]INFO 2026-07-24 01:00:30 ot_train.py:661 step:57K smpl:4M ep:83K epch:16.82 loss:0.015 grdn:0.201 lr:5.5e-07 updt_s:3.552 data_s:0.539 smp/s:16 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.005 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.119 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  76%|███████▌  | 7600/10000 [8:38:55<2:39:14,  3.98s/step]INFO 2026-07-24 01:14:27 ot_train.py:661 step:58K smpl:4M ep:83K epch:16.88 loss:0.014 grdn:0.205 lr:4.7e-07 updt_s:3.595 data_s:0.586 smp/s:15 mem_gb:29.85 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.004 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.128 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  78%|███████▊  | 7800/10000 [8:52:32<2:24:37,  3.94s/step]INFO 2026-07-24 01:28:03 ot_train.py:661 step:58K smpl:4M ep:84K epch:16.94 loss:0.014 grdn:0.199 lr:4.0e-07 updt_s:3.538 data_s:0.544 smp/s:16 mem_gb:29.85 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.004 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.134 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  80%|████████  | 8000/10000 [9:06:27<2:13:40,  4.01s/step]INFO 2026-07-24 01:41:59 ot_train.py:661 step:58K smpl:4M ep:84K epch:17.00 loss:0.015 grdn:0.209 lr:3.3e-07 updt_s:3.560 data_s:0.612 smp/s:15 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.003 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.131 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  82%|████████▏ | 8200/10000 [9:20:26<2:00:16,  4.01s/step]INFO 2026-07-24 01:55:57 ot_train.py:661 step:58K smpl:4M ep:84K epch:17.06 loss:0.015 grdn:0.202 lr:2.7e-07 updt_s:3.582 data_s:0.609 smp/s:15 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.002 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.130 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  84%|████████▍ | 8400/10000 [9:36:41<2:06:49,  4.76s/step]INFO 2026-07-24 02:12:12 ot_train.py:661 step:58K smpl:4M ep:84K epch:17.12 loss:0.014 grdn:0.193 lr:2.2e-07 updt_s:3.983 data_s:0.887 smp/s:13 mem_gb:29.84 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.002 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.127 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  86%|████████▌ | 8600/10000 [9:51:48<1:37:09,  4.16s/step]INFO 2026-07-24 02:27:20 ot_train.py:661 step:59K smpl:4M ep:85K epch:17.17 loss:0.015 grdn:0.212 lr:1.7e-07 updt_s:3.602 data_s:0.930 smp/s:14 mem_gb:29.84 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.002 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.149 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  88%|████████▊ | 8800/10000 [10:05:55<1:21:37,  4.08s/step]INFO 2026-07-24 02:41:27 ot_train.py:661 step:59K smpl:4M ep:85K epch:17.23 loss:0.015 grdn:0.202 lr:1.3e-07 updt_s:3.527 data_s:0.705 smp/s:15 mem_gb:29.83 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.001 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.132 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  90%|█████████ | 9000/10000 [10:20:01<1:15:04,  4.50s/step]INFO 2026-07-24 02:55:32 ot_train.py:661 step:59K smpl:4M ep:85K epch:17.29 loss:0.014 grdn:0.197 lr:9.2e-08 updt_s:3.561 data_s:0.662 smp/s:15 mem_gb:29.85 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.001 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.136 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  92%|█████████▏| 9200/10000 [10:34:22<59:02,  4.43s/step]INFO 2026-07-24 03:09:54 ot_train.py:661 step:59K smpl:4M ep:86K epch:17.35 loss:0.014 grdn:0.190 lr:6.2e-08 updt_s:3.625 data_s:0.681 smp/s:15 mem_gb:29.85 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.001 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.118 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  94%|█████████▍| 9400/10000 [10:50:58<59:54,  5.99s/step]  INFO 2026-07-24 03:26:29 ot_train.py:661 step:59K smpl:4M ep:86K epch:17.41 loss:0.015 grdn:0.206 lr:3.7e-08 updt_s:3.835 data_s:1.134 smp/s:13 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.000 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.130 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  96%|█████████▌| 9600/10000 [11:05:41<28:38,  4.30s/step]INFO 2026-07-24 03:41:13 ot_train.py:661 step:60K smpl:4M ep:86K epch:17.47 loss:0.015 grdn:0.200 lr:1.9e-08 updt_s:3.621 data_s:0.793 smp/s:15 mem_gb:29.85 flow_matching_loss:0.015 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.015 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.000 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.128 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training:  98%|█████████▊| 9800/10000 [11:19:44<14:04,  4.22s/step]INFO 2026-07-24 03:55:15 ot_train.py:661 step:60K smpl:4M ep:86K epch:17.53 loss:0.014 grdn:0.196 lr:7.1e-09 updt_s:3.584 data_s:0.626 smp/s:15 mem_gb:29.85 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.000 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.124 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+Training: 100%|██████████| 10000/10000 [11:33:57<00:00,  3.37s/step]INFO 2026-07-24 04:09:29 ot_train.py:661 step:60K smpl:4M ep:87K epch:17.59 loss:0.014 grdn:0.208 lr:1.0e-09 updt_s:3.553 data_s:0.712 smp/s:15 mem_gb:29.85 flow_matching_loss:0.014 rkd_loss:0.003 rkd_distance_loss:0.001 rkd_angle_loss:0.001 weighted_flow_matching_loss:0.014 weighted_rkd_loss:0.000 rkd_fm_loss_weight:1.000 rkd_loss_weight:0.000 rkd_runtime_enabled:1.000 rkd_student_pooling_id:1.000 rkd_teacher_latent_norm:1.000 rkd_student_feature_norm:15.128 training_phase_index:0.000 training_phase_start_step:0.000 training_phase_until_step:60000.000 training_phase_trainable_params:1080755329.000
+INFO 2026-07-24 04:09:29 ot_train.py:707 Checkpoint policy after step 60000
+INFO 2026-07-24 04:10:06 in_yaml.py:1414 Saved verified rolling checkpoint step=60000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01
+Training: 100%|██████████| 10000/10000 [11:34:35<00:00,  4.17s/step]
+INFO 2026-07-24 04:10:06 ot_train.py:793 End of training

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/files/requirements.txt

@@ -0,0 +1,118 @@
+lerobot==0.6.1
+tzdata==2026.3
+nvidia-cufile-cu12==1.13.1.3
+wandb==0.27.2
+six==1.17.0
+peft==0.19.1
+nvidia-curand-cu12==10.3.9.90
+nvidia-cuda-runtime-cu12==12.8.90
+datasets==4.8.5
+attrs==26.1.0
+cloudpickle==3.1.2
+packaging==25.0
+nvidia-cublas-cu12==12.8.4.1
+cmake==4.1.3
+psutil==7.2.2
+pydantic==2.13.4
+GitPython==3.1.51
+pillow==12.2.0
+pytz==2026.2
+cuda-pathfinder==1.2.2
+nvidia-cusparse-cu12==12.5.8.93
+typing_extensions==4.15.0
+urllib3==2.7.0
+pydantic_core==2.46.4
+draccus==0.10.0
+mypy_extensions==1.1.0
+dill==0.4.1
+typer==0.26.8
+decord==0.6.0
+tqdm==4.68.4
+triton==3.6.0
+idna==3.18
+requests==2.34.2
+nvidia-cudnn-cu12==9.19.0.56
+mdurl==0.1.2
+shellingham==1.5.4
+zipp==4.1.0
+timm==1.0.28
+torch==2.11.0+cu128
+tokenizers==0.22.2
+nvidia-cusparselt-cu12==0.7.1
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cusolver-cu12==11.7.3.90
+opencv-python-headless==4.13.0.92
+httpx==0.28.1
+Jinja2==3.1.6
+einops==0.8.2
+mergedeep==1.3.4
+platformdirs==4.10.0
+nvidia-nvjitlink-cu12==12.8.93
+lerobot==0.6.0
+PyYAML==6.0.3
+accelerate==1.14.0
+anyio==4.14.2
+gymnasium==1.3.0
+pip==26.1.2
+huggingface_hub==1.23.0
+diffusers==0.35.2
+annotated-doc==0.0.4
+pyarrow==25.0.0
+python-dateutil==2.9.0.post0
+termcolor==3.3.0
+nvidia-nccl-cu12==2.28.9
+sentry-sdk==2.65.0
+torchcodec==0.11.1
+pyyaml-include==1.4.1
+multiprocess==0.70.19
+wrapt==2.2.2
+typing-inspection==0.4.2
+nvidia-cuda-nvrtc-cu12==12.8.93
+Pygments==2.20.0
+nvidia-cuda-cupti-cu12==12.8.90
+certifi==2026.6.17
+setuptools==78.1.0
+charset-normalizer==3.4.9
+propcache==0.5.2
+multidict==6.7.1
+regex==2026.7.10
+frozenlist==1.8.0
+smmap==5.0.3
+hf-xet==1.5.1
+torchvision==0.26.0+cu128
+gitdb==4.0.12
+transformers==5.5.4
+rich==15.0.0
+cuda-bindings==12.9.4
+protobuf==7.35.1
+aiosignal==1.4.0
+jsonlines==4.0.0
+yarl==1.24.2
+filelock==3.29.0
+av==15.1.0
+networkx==3.6.1
+sympy==1.14.0
+wheel==0.47.0
+nvidia-nvtx-cu12==12.8.90
+xxhash==3.8.1
+toml==0.10.2
+MarkupSafe==3.0.3
+pandas==2.3.3
+safetensors==0.8.0
+click==8.4.2
+fsspec==2026.2.0
+dm-tree==0.1.10
+absl-py==2.5.0
+Farama-Notifications==0.0.6
+mpmath==1.3.0
+importlib_metadata==9.0.0
+markdown-it-py==4.2.0
+h11==0.16.0
+typing-inspect==0.9.0
+httpcore==1.0.9
+annotated-types==0.7.0
+numpy==2.2.6
+nvidia-nvshmem-cu12==3.4.5
+aiohappyeyeballs==2.7.1
+cuda-toolkit==12.8.1
+aiohttp==3.14.1

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/files/wandb-metadata.json

@@ -0,0 +1,46 @@
+{
+  "os": "Linux-6.8.0-55-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-23T16:35:01.611693Z",
+  "args": [
+    "--config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/train_config.json",
+    "--resume=true"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePath": "clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePathLocal": "clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K",
+  "host": "ext_minje-box",
+  "executable": "/home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 2,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "1327086579712"
+    }
+  },
+  "memory": {
+    "total": "2164176568320"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "3vx7sedbdsz3uay6fuuuh2e3qlqtbnr9"
+}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/training_phase_trainable_params":1.080755329e+09,"train/update_s":3.5525975197848676,"train/rkd_teacher_latent_norm":1,"train/rkd_student_pooling_id":1,"train/weighted_rkd_loss":2.6304401752318693e-08,"train/rkd_runtime_enabled":1,"train/episodes":86694.41811262691,"train/training_phase_index":0,"train/samples_per_s":15.006542098698269,"train/flow_matching_loss":0.014334074733778835,"train/epochs":17.58507466787564,"train/rkd_loss_weight":9.207125008732775e-06,"_runtime":83988,"train/lr":1.0049909226259037e-09,"train/loss":0.014334101057611405,"train/rkd_student_feature_norm":15.128039054870605,"_timestamp":1.7848661692599273e+09,"train/rkd_fm_loss_weight":1,"train/rkd_angle_loss":0.0010985716705908999,"_step":60000,"train/dataloading_s":0.712209094495156,"train/gpu_mem_gb":29.8462087392807,"train/steps":60000,"train/samples":3840000,"train/weighted_flow_matching_loss":0.014334074733778835,"_wandb":{"runtime":83988},"train/training_phase_start_step":0,"train/rkd_loss":0.0028473387891426684,"train/rkd_distance_loss":0.0006501954395207577,"train/training_phase_until_step":60000,"train/grad_norm":0.20803156722337007}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/logs/debug-core.log

@@ -0,0 +1,71 @@
+{"time":"2026-07-23T16:35:01.682970498Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpa9piztjm/port-1456.txt","pid":1456,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.682978333Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmplz0alzhu/port-1457.txt","pid":1457,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.682985044Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpwunx93cw/port-1458.txt","pid":1458,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.6829908Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpf4abll73/port-1459.txt","pid":1459,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.68360802Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1456}
+{"time":"2026-07-23T16:35:01.683616029Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1457}
+{"time":"2026-07-23T16:35:01.683634398Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1458}
+{"time":"2026-07-23T16:35:01.683603455Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1456-2506-1311874521/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.68361302Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1457-2507-772500566/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.683630442Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1458-2509-4014368169/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.683627313Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1459-2508-1604368573/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.683696179Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1459}
+{"time":"2026-07-23T16:35:01.832927644Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.833171958Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.833208155Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.833275727Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.837817931Z","level":"INFO","msg":"handleInformInit: received","streamId":"rs1egwv8","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.837959217Z","level":"INFO","msg":"handleInformInit: received","streamId":"2vl3enlf","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.838627327Z","level":"INFO","msg":"handleInformInit: received","streamId":"jeqgwsec","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.838419673Z","level":"INFO","msg":"handleInformInit: received","streamId":"x6da8gup","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.918297888Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpuyiti7pv/port-1460.txt","pid":1460,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.91863949Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1460}
+{"time":"2026-07-23T16:35:01.918628695Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1460-2829-1778593576/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:02.074586625Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.079634061Z","level":"INFO","msg":"handleInformInit: received","streamId":"brma7856","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.303790584Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2vl3enlf","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.30665649Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"x6da8gup","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.307741694Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"jeqgwsec","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.32303733Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rs1egwv8","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.493177984Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"brma7856","id":"1(@)"}
+{"time":"2026-07-23T16:35:08.314840933Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"pomu74zdcjwn"}
+{"time":"2026-07-23T16:35:08.415606838Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"u8kbql7lf4bh"}
+{"time":"2026-07-23T16:35:08.420996327Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"bhwn826heh2c"}
+{"time":"2026-07-23T16:35:08.623677864Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"0dnh3nypnhfm"}
+{"time":"2026-07-23T16:35:09.044152431Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4z6b40zq3yfu"}
+{"time":"2026-07-23T18:11:27.126647161Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.126779892Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T18:11:27.12683822Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1457-2507-772500566/socket","Net":"unix"}}
+{"time":"2026-07-23T18:11:27.126852003Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.126886112Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.126932287Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.378619144Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.37874002Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.378802617Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.378812716Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T18:11:27.378878838Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1456-2506-1311874521/socket","Net":"unix"}}
+{"time":"2026-07-23T18:11:27.378895674Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T18:11:35.494138091Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T18:11:35.494189389Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T18:11:35.494201536Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T18:12:28.994694228Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T18:12:28.994728432Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T18:12:28.994736061Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-24T04:08:45.417129571Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.417350121Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-24T04:08:45.417341706Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.417373499Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.41745467Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.417448026Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1458-2509-4014368169/socket","Net":"unix"}}
+{"time":"2026-07-24T04:08:47.515259401Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-24T04:08:47.515290273Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-24T04:08:47.515301151Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-24T04:10:06.98288621Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-24T04:10:06.983062676Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-24T04:10:06.983136342Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-24T04:10:06.983144044Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-24T04:10:06.98322588Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1459-2508-1604368573/socket","Net":"unix"}}
+{"time":"2026-07-24T04:10:06.983271294Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-24T04:10:23.060371611Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-24T04:10:23.060399527Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-24T04:10:23.060415126Z","level":"INFO","msg":"server is closed"}

Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/logs/debug.log

@@ -0,0 +1,65 @@
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_setup.py:_flush():81] Configure stats pid to 1459
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/logs/debug.log
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/wandb/run-20260723_163501-rs1egwv8/logs/debug-internal.log
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:init():766] calling init triggers
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'rs1egwv8', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01', '_wandb': {}}
+2026-07-23 16:35:01,612 INFO    MainThread:1459 [wandb_init.py:init():814] starting backend
+2026-07-23 16:35:01,833 INFO    MainThread:1459 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 16:35:02,323 INFO    MainThread:1459 [wandb_init.py:init():834] backend started and connected
+2026-07-23 16:35:02,325 INFO    MainThread:1459 [wandb_init.py:init():904] updated telemetry
+2026-07-23 16:35:02,326 INFO    MainThread:1459 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 16:35:03,342 INFO    MainThread:1459 [wandb_init.py:init():967] run resumed
+2026-07-23 16:35:03,344 INFO    MainThread:1459 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 16:35:03,396 INFO    MainThread:1459 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 16:35:03,398 INFO    MainThread:1459 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-24 04:10:06,982 INFO    wandb-AsyncioManager-main:1459 [service_client.py:_forward_responses():122] Reached EOF.
+2026-07-24 04:10:06,983 INFO    wandb-AsyncioManager-main:1459 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
+2026-07-24 04:10:06,983 ERROR   wandb-AsyncioManager-main:1459 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
+    await asyncio_compat.race(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
+    async with open_task_group(race=True) as tg:
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
+    await anext(self.gen)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
+    await task_group._wait_all(race=race, timeout=exit_timeout)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
+    raise exc
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
+    result = await handle.wait_async(timeout=None)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
+    response = await self._handle.wait_async(timeout=timeout)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
+    raise HandleAbandonedError()
+wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
+2026-07-24 04:10:14,759 ERROR   wandb-AsyncioManager-main:1459 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
+    should_stop = await self._check_should_stop()
+                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
+    handle = await self._interface.deliver_async(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
+    handle = await self._client.deliver(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
+    handle = self._mailbox.require_response(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
+    raise MailboxClosedError()
+wandb.sdk.mailbox.mailbox.MailboxClosedError