Add files using upload-large-folder tool

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_postprocessor.json

@@ -0,0 +1,23 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "groot_action_unpack_unnormalize_v2",
+      "config": {
+        "env_action_dim": 10,
+        "normalize_min_max": true,
+        "clip_normalized_action": true,
+        "libero_gripper_action": false,
+        "libero_gripper_binarize": true
+      },
+      "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_preprocessor.json

@@ -0,0 +1,78 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "groot_n1_7_pack_inputs_v1",
+      "config": {
+        "state_horizon": 1,
+        "action_horizon": 16,
+        "valid_action_horizon": 16,
+        "video_horizon": null,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "language_key": "task",
+        "formalize_language": true,
+        "embodiment_tag": "new_embodiment",
+        "embodiment_mapping": {
+          "oxe_droid_relative_eef_relative_joint": 24,
+          "xdof_relative_eef_relative_joint": 27,
+          "xdof_relative_eef_relative_joint_subtask": 27,
+          "real_g1_relative_eef_relative_joints": 25,
+          "real_r1_pro_sharpa_relative_eef": 26,
+          "real_r1_pro_sharpa_relative_eef_human": 26,
+          "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
+          "real_r1_pro_sharpa_relative_eef_mecka": 26,
+          "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
+          "simpler_env_google": 0,
+          "simpler_env_widowx": 1,
+          "libero_sim": 2,
+          "new_embodiment": 10
+        },
+        "normalize_min_max": true,
+        "state_dropout_prob": 0.0,
+        "clip_outliers": true,
+        "use_percentiles": false,
+        "video_modality_keys": null,
+        "raw_stats": null,
+        "modality_config": null
+      },
+      "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors"
+    },
+    {
+      "registry_name": "groot_n1_7_vlm_encode_v1",
+      "config": {
+        "model_name": "nvidia/Cosmos-Reason2-2B",
+        "image_crop_size": [
+          230,
+          230
+        ],
+        "image_target_size": [
+          256,
+          256
+        ],
+        "shortest_image_edge": null,
+        "crop_fraction": null,
+        "use_albumentations": false,
+        "letter_box_transform": false,
+        "device": "cuda"
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/train_config.json

@@ -0,0 +1,269 @@
+{
+    "dataset": {
+        "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+        "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "groot_rkd",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    16
+                ]
+            },
+            "observation.images.wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_shoulder": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.guide": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    10
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model",
+        "pretrained_revision": null,
+        "chunk_size": 16,
+        "n_action_steps": 16,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "IDENTITY",
+            "ACTION": "IDENTITY"
+        },
+        "base_model_path": "nvidia/GR00T-N1.7-3B",
+        "action_decode_transform": null,
+        "embodiment_tag": "new_embodiment",
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true,
+        "tune_top_llm_layers": 0,
+        "num_inference_timesteps": null,
+        "rtc_ramp_rate": null,
+        "use_flash_attention": false,
+        "use_relative_actions": false,
+        "relative_exclude_joints": [],
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-05,
+        "warmup_steps": null,
+        "warmup_ratio": 0.05,
+        "use_bf16": true,
+        "model_params_fp32": true,
+        "image_size": [
+            256,
+            256
+        ],
+        "tokenizer_assets_repo": null,
+        "lora_rank": 0,
+        "lora_alpha": 16,
+        "lora_dropout": 0.1,
+        "lora_full_model": false,
+        "video_backend": "decord",
+        "balance_dataset_weights": true,
+        "balance_trajectory_weights": true,
+        "dataset_paths": null,
+        "output_dir": "./tmp/gr00t",
+        "save_steps": 1000,
+        "max_steps": 10000,
+        "batch_size": 32,
+        "dataloader_num_workers": 8,
+        "report_to": "wandb",
+        "resume": false,
+        "action_expert_num_layers": 16,
+        "rkd_enabled": false,
+        "rkd_vae_checkpoint": null,
+        "rkd_student_pooling": "attention",
+        "rkd_student_attention_hidden_dim": 512,
+        "rkd_student_attention_output_dim": 512,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 0.1,
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 1.0
+    },
+    "reward_model": null,
+    "output_dir": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline",
+    "job_name": "INSIGHT_6D_DiT16_Baseline",
+    "resume": true,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 64,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 60000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-05,
+        "grad_clip_norm": 1.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "jeqgwsec",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01"
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/resolved_config.yaml

@@ -0,0 +1,26 @@
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  rkd_enabled: false
+seed: 42
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/16/Baseline
+job_name: INSIGHT_6D_DiT16_Baseline
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/source_config.yaml

@@ -0,0 +1,69 @@
+# INSIGHT GR00T N1.7 training config
+# INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
+insight:
+  checkpoint_interval: 5K
+  checkpoint_slots: 2
+  # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
+  action_space: ee_abs_rot6d
+  # Options: coarse | detailed
+  prompt_set: detailed
+  save_steps: ["10K", "30K", "60K"]
+
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
+
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
+  # VLM
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  # Action Expert
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  # RKD policy is used only to expose the shared DiT-depth setting.
+  # This remains the action-loss-only control.
+  rkd_enabled: false
+
+
+seed: 42
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: ${HOME}/groot_insight/Abs_6D/DiT_Layer/16/Baseline
+job_name: INSIGHT_6D_DiT16_Baseline
+
+wandb:
+  enable: true
+  disable_artifact: true
+
+# Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
+#
+# dataset:
+#   eval_split: 0.0
+#   image_transforms:
+#     enable: false  # default; set true only for train-image augmentation.
+#
+# policy:
+#   base_model_path: nvidia/GR00T-N1.7-3B
+#   embodiment_tag: new_embodiment
+#   use_relative_actions: false
+#   use_bf16: true
+#   repo_id: Whalswp/INSIGHT  # unused while push_to_hub is false
+#
+# save_checkpoint: true
+# use_policy_training_preset: true
+# eval_steps: 0
+# env_eval_freq: 0
+#
+# insight.prompt_set options:
+# coarse: High-level task text that requires following the visual guide.
+# detailed: Direction- and interaction-specific text, including rotation and push/pull details.

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/optimizer_param_groups.json

@@ -0,0 +1,353 @@
+[
+    {
+        "lr": 1.7322955673980678e-06,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 1.7322955673980678e-06,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310,
+            311,
+            312
+        ]
+    }
+]

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/scheduler_state.json

@@ -0,0 +1,18 @@
+{
+    "base_lrs": [
+        0.0001,
+        0.0001
+    ],
+    "last_epoch": 55000,
+    "_step_count": 55001,
+    "_is_initial": false,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        1.7322955673980678e-06,
+        1.7322955673980678e-06
+    ],
+    "lr_lambdas": [
+        null,
+        null
+    ]
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/training_step.json

@@ -0,0 +1,5 @@
+{
+    "step": 55000,
+    "num_processes": 1,
+    "batch_size": 64
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/action_space_manifest.json

@@ -0,0 +1,144 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d",
+  "action_views": [
+    "ee_abs_rot6d"
+  ],
+  "source_feature": "action.ee_abs_rot6d",
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    10
+  ],
+  "names": [
+    "x",
+    "y",
+    "z",
+    "r_col0_x",
+    "r_col0_y",
+    "r_col0_z",
+    "r_col1_x",
+    "r_col1_y",
+    "r_col1_z",
+    "gripper"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277
+    ]
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/config.json

@@ -0,0 +1,118 @@
+{
+    "type": "groot_rkd",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                16
+            ]
+        },
+        "observation.images.wrist": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.right_shoulder": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.guide": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                10
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model",
+    "pretrained_revision": null,
+    "chunk_size": 16,
+    "n_action_steps": 16,
+    "max_state_dim": 132,
+    "max_action_dim": 132,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "IDENTITY",
+        "ACTION": "IDENTITY"
+    },
+    "base_model_path": "nvidia/GR00T-N1.7-3B",
+    "action_decode_transform": null,
+    "embodiment_tag": "new_embodiment",
+    "tune_llm": false,
+    "tune_visual": false,
+    "tune_projector": true,
+    "tune_diffusion_model": true,
+    "tune_vlln": true,
+    "tune_top_llm_layers": 0,
+    "num_inference_timesteps": null,
+    "rtc_ramp_rate": null,
+    "use_flash_attention": false,
+    "use_relative_actions": false,
+    "relative_exclude_joints": [],
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-05,
+    "warmup_steps": null,
+    "warmup_ratio": 0.05,
+    "use_bf16": true,
+    "model_params_fp32": true,
+    "image_size": [
+        256,
+        256
+    ],
+    "tokenizer_assets_repo": null,
+    "lora_rank": 0,
+    "lora_alpha": 16,
+    "lora_dropout": 0.1,
+    "lora_full_model": false,
+    "video_backend": "decord",
+    "balance_dataset_weights": true,
+    "balance_trajectory_weights": true,
+    "dataset_paths": null,
+    "output_dir": "./tmp/gr00t",
+    "save_steps": 1000,
+    "max_steps": 10000,
+    "batch_size": 32,
+    "dataloader_num_workers": 8,
+    "report_to": "wandb",
+    "resume": false,
+    "action_expert_num_layers": 16,
+    "rkd_enabled": false,
+    "rkd_vae_checkpoint": null,
+    "rkd_student_pooling": "attention",
+    "rkd_student_attention_hidden_dim": 512,
+    "rkd_student_attention_output_dim": 512,
+    "rkd_fm_loss_weight": 1.0,
+    "rkd_loss_weight": 0.1,
+    "rkd_distance_weight": 1.0,
+    "rkd_angle_weight": 1.0
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_postprocessor.json

@@ -0,0 +1,23 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "groot_action_unpack_unnormalize_v2",
+      "config": {
+        "env_action_dim": 10,
+        "normalize_min_max": true,
+        "clip_normalized_action": true,
+        "libero_gripper_action": false,
+        "libero_gripper_binarize": true
+      },
+      "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_preprocessor.json

@@ -0,0 +1,78 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "groot_n1_7_pack_inputs_v1",
+      "config": {
+        "state_horizon": 1,
+        "action_horizon": 16,
+        "valid_action_horizon": 16,
+        "video_horizon": null,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "language_key": "task",
+        "formalize_language": true,
+        "embodiment_tag": "new_embodiment",
+        "embodiment_mapping": {
+          "oxe_droid_relative_eef_relative_joint": 24,
+          "xdof_relative_eef_relative_joint": 27,
+          "xdof_relative_eef_relative_joint_subtask": 27,
+          "real_g1_relative_eef_relative_joints": 25,
+          "real_r1_pro_sharpa_relative_eef": 26,
+          "real_r1_pro_sharpa_relative_eef_human": 26,
+          "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
+          "real_r1_pro_sharpa_relative_eef_mecka": 26,
+          "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
+          "simpler_env_google": 0,
+          "simpler_env_widowx": 1,
+          "libero_sim": 2,
+          "new_embodiment": 10
+        },
+        "normalize_min_max": true,
+        "state_dropout_prob": 0.0,
+        "clip_outliers": true,
+        "use_percentiles": false,
+        "video_modality_keys": null,
+        "raw_stats": null,
+        "modality_config": null
+      },
+      "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors"
+    },
+    {
+      "registry_name": "groot_n1_7_vlm_encode_v1",
+      "config": {
+        "model_name": "nvidia/Cosmos-Reason2-2B",
+        "image_crop_size": [
+          230,
+          230
+        ],
+        "image_target_size": [
+          256,
+          256
+        ],
+        "shortest_image_edge": null,
+        "crop_fraction": null,
+        "use_albumentations": false,
+        "letter_box_transform": false,
+        "device": "cuda"
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    }
+  ]
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/train_config.json

@@ -0,0 +1,269 @@
+{
+    "dataset": {
+        "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+        "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "groot_rkd",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    16
+                ]
+            },
+            "observation.images.wrist": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.right_shoulder": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.guide": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    10
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model",
+        "pretrained_revision": null,
+        "chunk_size": 16,
+        "n_action_steps": 16,
+        "max_state_dim": 132,
+        "max_action_dim": 132,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "IDENTITY",
+            "ACTION": "IDENTITY"
+        },
+        "base_model_path": "nvidia/GR00T-N1.7-3B",
+        "action_decode_transform": null,
+        "embodiment_tag": "new_embodiment",
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true,
+        "tune_top_llm_layers": 0,
+        "num_inference_timesteps": null,
+        "rtc_ramp_rate": null,
+        "use_flash_attention": false,
+        "use_relative_actions": false,
+        "relative_exclude_joints": [],
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-05,
+        "warmup_steps": null,
+        "warmup_ratio": 0.05,
+        "use_bf16": true,
+        "model_params_fp32": true,
+        "image_size": [
+            256,
+            256
+        ],
+        "tokenizer_assets_repo": null,
+        "lora_rank": 0,
+        "lora_alpha": 16,
+        "lora_dropout": 0.1,
+        "lora_full_model": false,
+        "video_backend": "decord",
+        "balance_dataset_weights": true,
+        "balance_trajectory_weights": true,
+        "dataset_paths": null,
+        "output_dir": "./tmp/gr00t",
+        "save_steps": 1000,
+        "max_steps": 10000,
+        "batch_size": 32,
+        "dataloader_num_workers": 8,
+        "report_to": "wandb",
+        "resume": false,
+        "action_expert_num_layers": 16,
+        "rkd_enabled": false,
+        "rkd_vae_checkpoint": null,
+        "rkd_student_pooling": "attention",
+        "rkd_student_attention_hidden_dim": 512,
+        "rkd_student_attention_output_dim": 512,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 0.1,
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 1.0
+    },
+    "reward_model": null,
+    "output_dir": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline",
+    "job_name": "INSIGHT_6D_DiT16_Baseline",
+    "resume": true,
+    "seed": 42,
+    "cudnn_deterministic": false,
+    "num_workers": 4,
+    "batch_size": 64,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "steps": 60000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-05,
+        "grad_clip_norm": 1.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": true,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "jeqgwsec",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01"
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/prompt_manifest.json

@@ -0,0 +1,40 @@
+{
+  "schema_version": 1,
+  "prompt_set": "detailed",
+  "source_file": "task_prompts.json",
+  "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
+  "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "task_count": 13,
+  "task_codes": [
+    "5f",
+    "5g",
+    "1ext",
+    "3b",
+    "5b",
+    "3a",
+    "3c",
+    "5h",
+    "5d",
+    "3d",
+    "5e",
+    "5a",
+    "5c"
+  ],
+  "prompts": {
+    "5f": "Close the bottle in clockwise direction.",
+    "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
+    "1ext": "Find the arrow guide and open the indicated drawer.",
+    "3b": "Open the door, rotate counter-clockwise and push.",
+    "5b": "Open the bottle in counter-clockwise direction.",
+    "3a": "Open the door, rotate clockwise and push.",
+    "3c": "Open the door, rotate clockwise and pull.",
+    "5h": "Close the bottle in counter-clockwise direction.",
+    "5d": "Close the bottle in clockwise direction.",
+    "3d": "Open the door, rotate counter-clockwise and pull.",
+    "5e": "Close the bottle in counter-clockwise direction.",
+    "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
+    "5c": "Open the bottle in clockwise direction."
+  }
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/resolved_config.yaml

@@ -0,0 +1,26 @@
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  rkd_enabled: false
+seed: 42
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/16/Baseline
+job_name: INSIGHT_6D_DiT16_Baseline
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/source_config.yaml

@@ -0,0 +1,69 @@
+# INSIGHT GR00T N1.7 training config
+# INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
+insight:
+  checkpoint_interval: 5K
+  checkpoint_slots: 2
+  # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
+  action_space: ee_abs_rot6d
+  # Options: coarse | detailed
+  prompt_set: detailed
+  save_steps: ["10K", "30K", "60K"]
+
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
+
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
+  # VLM
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  # Action Expert
+  action_expert_num_layers: 16
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  # RKD policy is used only to expose the shared DiT-depth setting.
+  # This remains the action-loss-only control.
+  rkd_enabled: false
+
+
+seed: 42
+batch_size: 64
+steps: 60000
+log_freq: 200
+output_dir: ${HOME}/groot_insight/Abs_6D/DiT_Layer/16/Baseline
+job_name: INSIGHT_6D_DiT16_Baseline
+
+wandb:
+  enable: true
+  disable_artifact: true
+
+# Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
+#
+# dataset:
+#   eval_split: 0.0
+#   image_transforms:
+#     enable: false  # default; set true only for train-image augmentation.
+#
+# policy:
+#   base_model_path: nvidia/GR00T-N1.7-3B
+#   embodiment_tag: new_embodiment
+#   use_relative_actions: false
+#   use_bf16: true
+#   repo_id: Whalswp/INSIGHT  # unused while push_to_hub is false
+#
+# save_checkpoint: true
+# use_policy_training_preset: true
+# eval_steps: 0
+# env_eval_freq: 0
+#
+# insight.prompt_set options:
+# coarse: High-level task text that requires following the visual guide.
+# detailed: Direction- and interaction-specific text, including rotation and push/pull details.

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/optimizer_param_groups.json

@@ -0,0 +1,353 @@
+[
+    {
+        "lr": 0.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39,
+            40,
+            41,
+            42,
+            43,
+            44,
+            45,
+            46,
+            47,
+            48,
+            49,
+            50,
+            51,
+            52,
+            53,
+            54,
+            55,
+            56,
+            57,
+            58,
+            59,
+            60,
+            61,
+            62,
+            63,
+            64,
+            65,
+            66,
+            67,
+            68,
+            69,
+            70,
+            71,
+            72,
+            73,
+            74,
+            75,
+            76,
+            77,
+            78,
+            79,
+            80,
+            81,
+            82,
+            83,
+            84,
+            85,
+            86,
+            87,
+            88,
+            89,
+            90,
+            91,
+            92,
+            93,
+            94,
+            95,
+            96,
+            97,
+            98,
+            99,
+            100,
+            101,
+            102,
+            103,
+            104,
+            105,
+            106,
+            107,
+            108,
+            109,
+            110,
+            111,
+            112,
+            113,
+            114,
+            115,
+            116,
+            117,
+            118,
+            119,
+            120,
+            121,
+            122,
+            123,
+            124,
+            125,
+            126,
+            127,
+            128,
+            129,
+            130,
+            131,
+            132,
+            133,
+            134,
+            135,
+            136,
+            137,
+            138,
+            139,
+            140,
+            141,
+            142,
+            143,
+            144,
+            145,
+            146,
+            147,
+            148,
+            149,
+            150,
+            151,
+            152,
+            153,
+            154
+        ]
+    },
+    {
+        "weight_decay": 0.0,
+        "lr": 0.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0001,
+        "params": [
+            155,
+            156,
+            157,
+            158,
+            159,
+            160,
+            161,
+            162,
+            163,
+            164,
+            165,
+            166,
+            167,
+            168,
+            169,
+            170,
+            171,
+            172,
+            173,
+            174,
+            175,
+            176,
+            177,
+            178,
+            179,
+            180,
+            181,
+            182,
+            183,
+            184,
+            185,
+            186,
+            187,
+            188,
+            189,
+            190,
+            191,
+            192,
+            193,
+            194,
+            195,
+            196,
+            197,
+            198,
+            199,
+            200,
+            201,
+            202,
+            203,
+            204,
+            205,
+            206,
+            207,
+            208,
+            209,
+            210,
+            211,
+            212,
+            213,
+            214,
+            215,
+            216,
+            217,
+            218,
+            219,
+            220,
+            221,
+            222,
+            223,
+            224,
+            225,
+            226,
+            227,
+            228,
+            229,
+            230,
+            231,
+            232,
+            233,
+            234,
+            235,
+            236,
+            237,
+            238,
+            239,
+            240,
+            241,
+            242,
+            243,
+            244,
+            245,
+            246,
+            247,
+            248,
+            249,
+            250,
+            251,
+            252,
+            253,
+            254,
+            255,
+            256,
+            257,
+            258,
+            259,
+            260,
+            261,
+            262,
+            263,
+            264,
+            265,
+            266,
+            267,
+            268,
+            269,
+            270,
+            271,
+            272,
+            273,
+            274,
+            275,
+            276,
+            277,
+            278,
+            279,
+            280,
+            281,
+            282,
+            283,
+            284,
+            285,
+            286,
+            287,
+            288,
+            289,
+            290,
+            291,
+            292,
+            293,
+            294,
+            295,
+            296,
+            297,
+            298,
+            299,
+            300,
+            301,
+            302,
+            303,
+            304,
+            305,
+            306,
+            307,
+            308,
+            309,
+            310,
+            311,
+            312
+        ]
+    }
+]

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/scheduler_state.json

@@ -0,0 +1,18 @@
+{
+    "base_lrs": [
+        0.0001,
+        0.0001
+    ],
+    "last_epoch": 60000,
+    "_step_count": 60001,
+    "_is_initial": false,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+        0.0,
+        0.0
+    ],
+    "lr_lambdas": [
+        null,
+        null
+    ]
+}

Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/training_step.json

@@ -0,0 +1,5 @@
+{
+    "step": 60000,
+    "num_processes": 1,
+    "batch_size": 64
+}

Abs_6D/DiT_Layer/16/Baseline/wandb/debug.log

@@ -1,20 +1,65 @@
-2026-07-23 01:33:42,279 INFO    MainThread:814713 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
-2026-07-23 01:33:42,279 INFO    MainThread:814713 [wandb_setup.py:_flush():81] Configure stats pid to 814713
-2026-07-23 01:33:42,280 INFO    MainThread:814713 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-07-23 01:33:42,280 INFO    MainThread:814713 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug.log
-2026-07-23 01:33:42,280 INFO    MainThread:814713 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-internal.log
-2026-07-23 01:33:42,280 INFO    MainThread:814713 [wandb_init.py:init():766] calling init triggers
-2026-07-23 01:33:42,280 INFO    MainThread:814713 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
-config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'svym8k0w', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000', '_wandb': {}}
-2026-07-23 01:33:42,280 INFO    MainThread:814713 [wandb_init.py:init():814] starting backend
-2026-07-23 01:33:42,497 INFO    MainThread:814713 [wandb_init.py:init():829] sending inform_init request
-2026-07-23 01:33:42,962 INFO    MainThread:814713 [wandb_init.py:init():834] backend started and connected
-2026-07-23 01:33:42,964 INFO    MainThread:814713 [wandb_init.py:init():904] updated telemetry
-2026-07-23 01:33:42,969 INFO    MainThread:814713 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
-2026-07-23 01:33:43,656 INFO    MainThread:814713 [wandb_init.py:init():967] run resumed
-2026-07-23 01:33:43,659 INFO    MainThread:814713 [wandb_init.py:init():972] starting run threads in backend
-2026-07-23 01:33:43,720 INFO    MainThread:814713 [wandb_run.py:_console_start():2523] atexit reg
-2026-07-23 01:33:43,720 INFO    MainThread:814713 [wandb_run.py:_redirect():2373] redirect: wrap_raw
-2026-07-23 01:33:43,720 INFO    MainThread:814713 [wandb_run.py:_redirect():2442] Wrapping output streams.
-2026-07-23 01:33:43,720 INFO    MainThread:814713 [wandb_run.py:_redirect():2465] Redirects installed.
-2026-07-23 01:33:43,722 INFO    MainThread:814713 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_setup.py:_flush():81] Configure stats pid to 1458
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/logs/debug.log
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/logs/debug-internal.log
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_init.py:init():766] calling init triggers
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'jeqgwsec', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01', '_wandb': {}}
+2026-07-23 16:35:01,612 INFO    MainThread:1458 [wandb_init.py:init():814] starting backend
+2026-07-23 16:35:01,833 INFO    MainThread:1458 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 16:35:02,308 INFO    MainThread:1458 [wandb_init.py:init():834] backend started and connected
+2026-07-23 16:35:02,310 INFO    MainThread:1458 [wandb_init.py:init():904] updated telemetry
+2026-07-23 16:35:02,310 INFO    MainThread:1458 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 16:35:03,987 INFO    MainThread:1458 [wandb_init.py:init():967] run resumed
+2026-07-23 16:35:03,989 INFO    MainThread:1458 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 16:35:04,041 INFO    MainThread:1458 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 16:35:04,041 INFO    MainThread:1458 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 16:35:04,041 INFO    MainThread:1458 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 16:35:04,041 INFO    MainThread:1458 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 16:35:04,043 INFO    MainThread:1458 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-24 04:08:45,417 INFO    wandb-AsyncioManager-main:1458 [service_client.py:_forward_responses():122] Reached EOF.
+2026-07-24 04:08:45,417 INFO    wandb-AsyncioManager-main:1458 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
+2026-07-24 04:08:45,418 ERROR   wandb-AsyncioManager-main:1458 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
+    await asyncio_compat.race(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
+    async with open_task_group(race=True) as tg:
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
+    await anext(self.gen)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
+    await task_group._wait_all(race=race, timeout=exit_timeout)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
+    raise exc
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
+    result = await handle.wait_async(timeout=None)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
+    response = await self._handle.wait_async(timeout=timeout)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
+    raise HandleAbandonedError()
+wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
+2026-07-24 04:08:45,582 ERROR   wandb-AsyncioManager-main:1458 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
+    should_stop = await self._check_should_stop()
+                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
+    handle = await self._interface.deliver_async(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
+    handle = await self._client.deliver(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
+    handle = self._mailbox.require_response(request)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
+    raise MailboxClosedError()
+wandb.sdk.mailbox.mailbox.MailboxClosedError

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/config.yaml

@@ -0,0 +1,334 @@
+_wandb:
+    value:
+        cli_version: 0.27.2
+        e:
+            xf8nsz01flzs1n2ps78z3eqf7rnh8bxr:
+                args:
+                    - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model/train_config.json
+                    - --resume=true
+                    - --output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline
+                    - --dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+                codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                codePathLocal: my_scripts/lerobot_train_yaml.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "1888425144320"
+                        used: "1137071325184"
+                executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python
+                git:
+                    commit: a329728c2c9524d79b966c4a7a82ee56a4c2abf7
+                    remote: git@github.com:paragon7060/clvla.git
+                gpu: NVIDIA H200
+                gpu_count: 2
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc
+                host: ext_minje-box
+                memory:
+                    total: "2164176568320"
+                os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39
+                program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                python: CPython 3.12.13
+                root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline
+                startedAt: "2026-07-23T01:33:42.278848Z"
+                writerId: xf8nsz01flzs1n2ps78z3eqf7rnh8bxr
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.12.13
+            "5": 0.27.2
+            "6": 5.5.4
+            "10":
+                - 21
+            "12": 0.27.2
+            "13": linux-x86_64
+batch_size:
+    value: 64
+checkpoint_path:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000
+cudnn_deterministic:
+    value: false
+dataset:
+    value:
+        depth_output_unit: mm
+        episodes: null
+        eval_split: 0
+        image_transforms:
+            enable: false
+            max_num_transforms: 3
+            random_order: false
+            tfs:
+                affine:
+                    kwargs:
+                        degrees:
+                            - -5
+                            - 5
+                        translate:
+                            - 0.05
+                            - 0.05
+                    type: RandomAffine
+                    weight: 1
+                brightness:
+                    kwargs:
+                        brightness:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                contrast:
+                    kwargs:
+                        contrast:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                hue:
+                    kwargs:
+                        hue:
+                            - -0.05
+                            - 0.05
+                    type: ColorJitter
+                    weight: 1
+                saturation:
+                    kwargs:
+                        saturation:
+                            - 0.5
+                            - 1.5
+                    type: ColorJitter
+                    weight: 1
+                sharpness:
+                    kwargs:
+                        sharpness:
+                            - 0.5
+                            - 1.5
+                    type: SharpnessJitter
+                    weight: 1
+        repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+        return_uint8: false
+        revision: null
+        root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+        streaming: false
+        use_imagenet_stats: true
+        video_backend: torchcodec
+env:
+    value: null
+env_eval_freq:
+    value: 20000
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        recording: false
+        recording_private: false
+        recording_repo_id: null
+        use_async_envs: true
+eval_steps:
+    value: 0
+job:
+    value:
+        detach: false
+        image: huggingface/lerobot-gpu:latest
+        tags: []
+        target: null
+        timeout: 2d
+job_name:
+    value: INSIGHT_6D_DiT16_Baseline
+log_freq:
+    value: 200
+max_eval_samples:
+    value: 0
+num_workers:
+    value: 4
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.999
+        eps: 1e-08
+        grad_clip_norm: 1
+        lr: 0.0001
+        type: adamw
+        weight_decay: 1e-05
+output_dir:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline
+peft:
+    value: null
+persistent_workers:
+    value: true
+policy:
+    value:
+        action_decode_transform: null
+        action_expert_num_layers: 16
+        balance_dataset_weights: true
+        balance_trajectory_weights: true
+        base_model_path: nvidia/GR00T-N1.7-3B
+        batch_size: 32
+        chunk_size: 16
+        dataloader_num_workers: 8
+        dataset_paths: null
+        device: cuda
+        embodiment_tag: new_embodiment
+        image_size:
+            - 256
+            - 256
+        input_features:
+            observation.images.guide:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.right_shoulder:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.wrist:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.state:
+                shape:
+                    - 16
+                type: STATE
+        license: null
+        lora_alpha: 16
+        lora_dropout: 0.1
+        lora_full_model: false
+        lora_rank: 0
+        max_action_dim: 132
+        max_state_dim: 132
+        max_steps: 10000
+        model_params_fp32: true
+        n_action_steps: 16
+        n_obs_steps: 1
+        normalization_mapping:
+            ACTION: IDENTITY
+            STATE: IDENTITY
+            VISUAL: IDENTITY
+        num_inference_timesteps: null
+        optimizer_betas:
+            - 0.9
+            - 0.999
+        optimizer_eps: 1e-08
+        optimizer_lr: 0.0001
+        optimizer_weight_decay: 1e-05
+        output_dir: ./tmp/gr00t
+        output_features:
+            action:
+                shape:
+                    - 10
+                type: ACTION
+        pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model
+        pretrained_revision: null
+        private: null
+        push_to_hub: false
+        relative_exclude_joints: []
+        repo_id: null
+        report_to: wandb
+        resume: false
+        rkd_angle_weight: 1
+        rkd_distance_weight: 1
+        rkd_enabled: false
+        rkd_fm_loss_weight: 1
+        rkd_loss_weight: 0.1
+        rkd_student_attention_hidden_dim: 512
+        rkd_student_attention_output_dim: 512
+        rkd_student_pooling: attention
+        rkd_vae_checkpoint: null
+        rtc_ramp_rate: null
+        save_steps: 1000
+        tags: null
+        tokenizer_assets_repo: null
+        tune_diffusion_model: true
+        tune_llm: false
+        tune_projector: true
+        tune_top_llm_layers: 0
+        tune_visual: false
+        tune_vlln: true
+        type: groot_rkd
+        use_amp: false
+        use_bf16: true
+        use_flash_attention: false
+        use_peft: false
+        use_relative_actions: false
+        video_backend: decord
+        warmup_ratio: 0.05
+        warmup_steps: null
+prefetch_factor:
+    value: 4
+resume:
+    value: true
+reward_model:
+    value: null
+sample_weighting:
+    value: null
+save_checkpoint:
+    value: true
+save_checkpoint_to_hub:
+    value: false
+save_freq:
+    value: 20000
+scheduler:
+    value:
+        name: cosine
+        num_warmup_steps: 500
+        type: diffuser
+seed:
+    value: 42
+steps:
+    value: 60000
+tolerance_s:
+    value: 0.0001
+use_policy_training_preset:
+    value: true
+wandb:
+    value:
+        add_tags: true
+        disable_artifact: true
+        enable: true
+        entity: null
+        mode: null
+        notes: null
+        project: lerobot
+        run_id: svym8k0w

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/output.log

@@ -45,4 +45,46 @@ Training:   3%|▎         | 800/30000 [20:32<12:38:00,  1.56s/step]INFO 2026-07
 Training:   3%|▎         | 1000/30000 [26:16<21:46:15,  2.70s/step]INFO 2026-07-23 02:00:27 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.09 loss:0.042 grdn:0.279 lr:4.8e-05 updt_s:0.840 data_s:0.878 smp/s:37 mem_gb:29.76
 Training:   4%|▍         | 1200/30000 [31:32<10:00:36,  1.25s/step]INFO 2026-07-23 02:05:43 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.14 loss:0.044 grdn:0.297 lr:4.8e-05 updt_s:0.810 data_s:0.768 smp/s:41 mem_gb:29.76
 Training:   5%|▍         | 1400/30000 [36:05<10:58:06,  1.38s/step]INFO 2026-07-23 02:10:17 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.20 loss:0.042 grdn:0.283 lr:4.7e-05 updt_s:0.744 data_s:0.623 smp/s:47 mem_gb:29.76
-Training:   5%|▍         | 1485/30000 [38:08<11:09:17,  1.41s/step]
+Training:   5%|▌         | 1600/30000 [40:59<18:07:32,  2.30s/step]INFO 2026-07-23 02:15:10 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.26 loss:0.042 grdn:0.294 lr:4.7e-05 updt_s:0.790 data_s:0.675 smp/s:44 mem_gb:29.75
+Training:   6%|▌         | 1800/30000 [45:49<10:55:53,  1.40s/step]INFO 2026-07-23 02:20:00 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.32 loss:0.040 grdn:0.283 lr:4.6e-05 updt_s:0.779 data_s:0.671 smp/s:44 mem_gb:29.75
+Training:   7%|▋         | 2000/30000 [50:37<10:47:26,  1.39s/step]INFO 2026-07-23 02:24:48 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.38 loss:0.039 grdn:0.285 lr:4.6e-05 updt_s:0.779 data_s:0.658 smp/s:45 mem_gb:29.76
+Training:   7%|▋         | 2169/30000 [54:38<10:42:52,  1.39s/step]Traceback (most recent call last):
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1453, in <module>
+    config_path = Path(config_value).expanduser()
+    ^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main
+    config_value = _find_cli_value(args, "--config_path")
+    ^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 813, in main
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 615, in train
+    wandb_logger.log_dict(wandb_log_dict, step)
+                     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 196, in update_policy
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/accelerate/optimizer.py", line 179, in step
+    self.optimizer.step(closure)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/lr_scheduler.py", line 166, in wrapper
+    return func.__get__(opt, opt.__class__)(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 533, in wrapper
+    out = func(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 81, in _use_grad
+    ret = func(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/adam.py", line 248, in step
+    adam(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 151, in maybe_fallback
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/adam.py", line 971, in adam
+    func(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/adam.py", line 773, in _multi_tensor_adam
+    1 - beta1 ** _get_value(step) for step in device_state_steps
+                 ^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 96, in _get_value
+    return x.item() if isinstance(x, torch.Tensor) else x
+           ^^^^^^^^
+KeyboardInterrupt

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/episodes":53461.55783611993,"_timestamp":1.784740277496143e+09,"train/steps":37000,"_wandb":{"runtime":58892},"train/samples":2368000,"train/loss":0.030536407604813576,"train/lr":3.280139452204795e-05,"train/epochs":10.844129378523313,"train/gpu_mem_gb":29.777451770305632,"train/samples_per_s":50.02997034996543,"train/dataloading_s":0.5564608199794747,"_step":37000,"train/update_s":0.7227723986755155,"train/grad_norm":0.2982970765978098,"_runtime":58892}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-core.log

@@ -26,3 +26,39 @@
 {"time":"2026-07-23T01:33:48.72926561Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"0uuooyt4of4j"}
 {"time":"2026-07-23T01:33:48.824026381Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"v9b9occvkv0z"}
 {"time":"2026-07-23T01:33:48.928044163Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3mgxscqyzkyn"}
+{"time":"2026-07-23T02:28:50.768484688Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768650257Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768660338Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768702111Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768725726Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768737691Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.768728092Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.768751955Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768745569Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768805358Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.768815828Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814707-815594-198914184/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:50.76883043Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814713-815525-1635480438/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:50.770084447Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.770158191Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.770173329Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.770146502Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.770246375Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814719-815551-3138903381/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:50.770382899Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773003008Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773054953Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773111864Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773106808Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:28:50.773073579Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:28:50.773200905Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814701-815561-915654594/socket","Net":"unix"}}
+{"time":"2026-07-23T02:28:52.626908382Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.626929775Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.626940482Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:28:52.662753611Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.662779647Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.662788745Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:28:52.68387658Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.683943364Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:28:52.683953617Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:29:02.470789399Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:29:02.470826347Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:29:02.470833873Z","level":"INFO","msg":"server is closed"}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-internal.log

@@ -322,3 +322,145 @@
 {"time":"2026-07-23T02:12:00.967408237Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-07-23T02:12:15.571990351Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":306,"events_lines":2,"console_offset":47,"console_lines":1}
 {"time":"2026-07-23T02:12:15.99962508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:12:30.572077763Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":308,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:12:30.919043368Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:12:45.571767312Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":310,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:12:46.019504323Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:13:00.57253239Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":312,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:13:01.087265906Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:13:15.571983877Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":314,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:13:15.974384985Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:13:30.572251144Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":316,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:13:30.977173841Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:13:45.57221059Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":318,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:13:46.241316548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:00.572670577Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":320,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:01.023306593Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:15.57176854Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":322,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:16.132358342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:30.57242089Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":324,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:31.099023189Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:14:45.57165267Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":326,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:14:46.083851981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:15:00.571775386Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":328,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-07-23T02:15:01.180994763Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:15:10.34397278Z","level":"WARN","msg":"handler: ignoring partial history record","step":31600,"current":37001}
+{"time":"2026-07-23T02:15:15.572476294Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":330,"events_lines":2,"console_offset":47,"console_lines":2}
+{"time":"2026-07-23T02:15:16.071355954Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:15:30.574107589Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":332,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:15:32.072341973Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:15:45.572433513Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":334,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:15:46.035308642Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:16:00.572615706Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":336,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:16:01.045603345Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:16:15.572099645Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":338,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:16:16.033118322Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:16:30.571789981Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":340,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:16:30.973875432Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:16:45.572039256Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":342,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:16:50.678188665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:17:00.572648582Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":344,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:17:01.118492107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:17:15.571969771Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":346,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:17:15.987739434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:17:30.571939605Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":348,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:17:31.006737517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:17:45.571815157Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":350,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:17:46.065856811Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:18:00.572429836Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":352,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:18:05.316116771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:18:15.571696271Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":354,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:18:16.048072889Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:18:30.572417546Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":356,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:18:31.020240289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:18:45.572300055Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":358,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:18:46.018867761Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:00.572596087Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":360,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:01.133577158Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:15.572022604Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":362,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:16.123757993Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:30.57193837Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":364,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:31.144756012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:19:45.572191385Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":366,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:19:46.029360586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:00.572190728Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":368,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-07-23T02:20:00.624607888Z","level":"WARN","msg":"handler: ignoring partial history record","step":31800,"current":37001}
+{"time":"2026-07-23T02:20:01.06336733Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:15.571825472Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":370,"events_lines":2,"console_offset":48,"console_lines":2}
+{"time":"2026-07-23T02:20:16.050034803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:30.571866542Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":372,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:20:31.077058719Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:20:45.572297048Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":374,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:20:46.099056398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:21:00.572442729Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":376,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:21:01.039290608Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:21:15.572106859Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":378,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:21:16.119198366Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:21:30.574065633Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":380,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:21:31.057563939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:21:45.571889035Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":382,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:21:45.991763188Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:22:00.572347747Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":384,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:22:01.11877397Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:22:15.571676227Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":386,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:22:16.195466356Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:22:30.572561938Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":388,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:22:31.156203527Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:22:45.571953289Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":390,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:22:46.096453182Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:23:00.572406554Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":392,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:23:01.15280111Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:23:15.572476772Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":394,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:23:15.997661649Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:23:30.571675416Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":396,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:23:31.042153827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:23:45.571842581Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":398,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:23:46.013948114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:00.572618553Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":400,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:24:01.112804396Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:15.571653775Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":402,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:24:19.116676651Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:30.571793404Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":404,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:24:31.111814247Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:45.572614744Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":406,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-07-23T02:24:46.108966733Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:24:48.481542966Z","level":"WARN","msg":"handler: ignoring partial history record","step":32000,"current":37001}
+{"time":"2026-07-23T02:25:00.572449255Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":408,"events_lines":2,"console_offset":49,"console_lines":2}
+{"time":"2026-07-23T02:25:01.074339199Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:25:15.572461247Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":410,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:25:19.46720351Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:25:30.572713746Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":412,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:25:31.036352395Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:25:45.572202664Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":414,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:25:46.095330102Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:26:00.572057444Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":416,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:26:00.997437674Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:26:15.572572229Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":418,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:26:15.964506584Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:26:30.572311377Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":420,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:26:34.223228815Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:26:45.571673127Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":422,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:26:45.981568849Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:00.572075585Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":424,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:05.612198431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:15.571939538Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":426,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:16.068326991Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:30.572025149Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":428,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:36.185985361Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:27:45.572361632Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":430,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:27:45.965826186Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:28:00.572268613Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":432,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:28:01.061533852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:28:15.572253679Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":434,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:28:19.240182832Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:28:30.57631386Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":436,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:28:34.507927646Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:28:45.571968022Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":438,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-07-23T02:28:45.957846418Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:28:52.099263267Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-07-23T02:28:52.099519179Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":50,"console_lines":40,"uploaded_len":3,"complete":true,"exit_code":255}
+{"time":"2026-07-23T02:28:52.662183615Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-23T02:28:52.662368771Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-07-23T02:28:52.662380976Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-07-23T02:28:52.662460976Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-07-23T02:28:52.662465356Z","level":"INFO","msg":"stream: all finished"}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug.log

@@ -18,3 +18,29 @@ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2'
 2026-07-23 01:33:43,720 INFO    MainThread:814713 [wandb_run.py:_redirect():2442] Wrapping output streams.
 2026-07-23 01:33:43,720 INFO    MainThread:814713 [wandb_run.py:_redirect():2465] Redirects installed.
 2026-07-23 01:33:43,722 INFO    MainThread:814713 [wandb_init.py:init():1010] run started, returning control to user process
+2026-07-23 02:28:50,768 INFO    wandb-AsyncioManager-main:814713 [service_client.py:_forward_responses():122] Reached EOF.
+2026-07-23 02:28:50,768 INFO    wandb-AsyncioManager-main:814713 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
+2026-07-23 02:28:50,769 ERROR   wandb-AsyncioManager-main:814713 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
+    await fn()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
+    await asyncio_compat.race(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
+    async with open_task_group(race=True) as tg:
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
+    await anext(self.gen)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
+    await task_group._wait_all(race=race, timeout=exit_timeout)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
+    raise exc
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
+    result = await handle.wait_async(timeout=None)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
+    response = await self._handle.wait_async(timeout=timeout)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
+    raise HandleAbandonedError()
+wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":0,"_wandb":{"runtime":0}}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-core.log

@@ -0,0 +1,30 @@
+{"time":"2026-07-23T02:29:37.218272849Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpbi3491l3/port-2499418.txt","pid":2499418,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T02:29:37.218902899Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2499418}
+{"time":"2026-07-23T02:29:37.218864854Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2499418-2500321-1544228546/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:37.402297125Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T02:29:37.409066123Z","level":"INFO","msg":"handleInformInit: received","streamId":"eb0sb786","id":"1(@)"}
+{"time":"2026-07-23T02:29:37.868916316Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"eb0sb786","id":"1(@)"}
+{"time":"2026-07-23T02:29:37.870710243Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpewnffvrr/port-2499551.txt","pid":2499551,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T02:29:37.871145347Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2499551-2500408-2757815951/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:37.871202129Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2499551}
+{"time":"2026-07-23T02:29:38.054483267Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.064116376Z","level":"INFO","msg":"handleInformInit: received","streamId":"cemf0rge","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.350897746Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351103797Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351133399Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:29:38.351150646Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351193796Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.351238698Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2499418-2500321-1544228546/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:38.352704212Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.352724918Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.352730597Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T02:29:38.445119298Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"cemf0rge","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796444376Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796598651Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796618722Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T02:29:38.796624128Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796668926Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.796751175Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2499551-2500408-2757815951/socket","Net":"unix"}}
+{"time":"2026-07-23T02:29:38.797992882Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.798016256Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T02:29:38.798022072Z","level":"INFO","msg":"server is closed"}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-07-23T02:29:37.409217201Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-23T02:29:37.40945435Z","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-23T02:29:37.868642986Z","level":"INFO","msg":"stream: created new stream","id":"eb0sb786"}
+{"time":"2026-07-23T02:29:37.868729478Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-23T02:29:37.868907916Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-23T02:29:37.868962463Z","level":"INFO","msg":"writer: started","stream_id":"eb0sb786"}
+{"time":"2026-07-23T02:29:37.868991239Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-23T02:29:38.34120831Z","level":"ERROR","msg":"runupserter: failed to init run","error":"no data but must resume"}
+{"time":"2026-07-23T02:29:38.351341243Z","level":"ERROR","msg":"sender: uploadConfigFile: runhandle: run not yet initialized"}
+{"time":"2026-07-23T02:29:38.351613477Z","level":"ERROR","msg":"runfiles: runhandle: run not yet initialized"}
+{"time":"2026-07-23T02:29:38.351689734Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-07-23T02:29:38.351697183Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-07-23T02:29:38.351703469Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-07-23T02:29:38.351730985Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-07-23T02:29:38.351733862Z","level":"INFO","msg":"stream: all finished"}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug.log

@@ -0,0 +1,13 @@
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_setup.py:_flush():81] Configure stats pid to 2499418
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug.log
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-internal.log
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_init.py:init():766] calling init triggers
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'eb0sb786', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000', '_wandb': {}}
+2026-07-23 02:29:37,186 INFO    MainThread:2499418 [wandb_init.py:init():814] starting backend
+2026-07-23 02:29:37,402 INFO    MainThread:2499418 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 02:29:37,869 INFO    MainThread:2499418 [wandb_init.py:init():834] backend started and connected
+2026-07-23 02:29:37,871 INFO    MainThread:2499418 [wandb_init.py:init():904] updated telemetry
+2026-07-23 02:29:37,875 INFO    MainThread:2499418 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/files/output.log

@@ -0,0 +1,149 @@
+INFO 2026-07-23 02:31:25 db_utils.py:126 Logs will be synced with wandb.
+INFO 2026-07-23 02:31:25 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/jeqgwsec
+INFO 2026-07-23 02:31:25 ot_train.py:307 Creating dataset
+INFO 2026-07-23 02:31:29 ot_train.py:341 Creating policy
+INFO 2026-07-23 02:31:29 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model
+INFO 2026-07-23 02:31:29 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 4284.76it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 807/807 [00:00<00:00, 1691.21it/s]
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                  | Status     |  |
+---------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{16...31}.ff.net.2.weight       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.bias     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.2.bias         | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.weight     | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+Loading weights from local directory
+INFO 2026-07-23 02:31:44 ot_train.py:414 Creating optimizer and scheduler
+INFO 2026-07-23 02:31:46 ot_train.py:446 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline
+INFO 2026-07-23 02:31:46 ot_train.py:453 cfg.steps=60000 (60K)
+INFO 2026-07-23 02:31:46 ot_train.py:454 dataset.num_frames=218367 (218K)
+INFO 2026-07-23 02:31:46 ot_train.py:455 dataset.num_episodes=4930
+INFO 2026-07-23 02:31:46 ot_train.py:458 Effective batch size: 64 x 1 = 64
+INFO 2026-07-23 02:31:46 ot_train.py:459 num_learnable_params=1079180416 (1B)
+INFO 2026-07-23 02:31:46 ot_train.py:460 num_total_params=2604255361 (3B)
+INFO 2026-07-23 02:31:46 ot_train.py:501 Resuming data order at epoch 13, sample 42176
+Training:   0%|          | 0/30000 [00:00<?, ?step/s]INFO 2026-07-23 02:31:46 ot_train.py:611 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   1%|          | 200/30000 [05:05<11:36:11,  1.40s/step]INFO 2026-07-23 02:36:52 ot_train.py:655 step:30K smpl:2M ep:44K epch:8.85 loss:0.042 grdn:0.277 lr:5.0e-05 updt_s:0.811 data_s:0.714 smp/s:42 mem_gb:29.76
+Training:   1%|▏         | 400/30000 [09:59<12:01:16,  1.46s/step]INFO 2026-07-23 02:41:46 ot_train.py:655 step:30K smpl:2M ep:44K epch:8.91 loss:0.041 grdn:0.281 lr:5.0e-05 updt_s:0.761 data_s:0.707 smp/s:44 mem_gb:29.76
+Training:   2%|▏         | 600/30000 [14:53<11:31:47,  1.41s/step]INFO 2026-07-23 02:46:39 ot_train.py:655 step:31K smpl:2M ep:44K epch:8.97 loss:0.043 grdn:0.301 lr:4.9e-05 updt_s:0.777 data_s:0.690 smp/s:44 mem_gb:29.76
+Training:   3%|▎         | 800/30000 [19:51<12:43:53,  1.57s/step]INFO 2026-07-23 02:51:38 ot_train.py:655 step:31K smpl:2M ep:45K epch:9.03 loss:0.042 grdn:0.285 lr:4.9e-05 updt_s:0.740 data_s:0.750 smp/s:43 mem_gb:29.76
+Training:   3%|▎         | 1000/30000 [26:17<29:03:36,  3.61s/step]INFO 2026-07-23 02:58:04 ot_train.py:655 step:31K smpl:2M ep:45K epch:9.09 loss:0.042 grdn:0.279 lr:4.8e-05 updt_s:0.768 data_s:1.161 smp/s:33 mem_gb:29.76
+Training:   4%|▍         | 1200/30000 [34:22<16:31:13,  2.07s/step]INFO 2026-07-23 03:06:09 ot_train.py:655 step:31K smpl:2M ep:45K epch:9.14 loss:0.044 grdn:0.297 lr:4.8e-05 updt_s:0.791 data_s:1.634 smp/s:26 mem_gb:29.76
+Training:   5%|▍         | 1400/30000 [40:30<12:59:24,  1.64s/step]INFO 2026-07-23 03:12:17 ot_train.py:655 step:31K smpl:2M ep:45K epch:9.20 loss:0.042 grdn:0.283 lr:4.7e-05 updt_s:0.780 data_s:1.056 smp/s:35 mem_gb:29.76
+Training:   5%|▌         | 1600/30000 [46:11<22:02:05,  2.79s/step]INFO 2026-07-23 03:17:58 ot_train.py:655 step:32K smpl:2M ep:46K epch:9.26 loss:0.042 grdn:0.294 lr:4.7e-05 updt_s:0.814 data_s:0.890 smp/s:38 mem_gb:29.75
+Training:   6%|▌         | 1800/30000 [54:35<24:33:28,  3.14s/step]INFO 2026-07-23 03:26:22 ot_train.py:655 step:32K smpl:2M ep:46K epch:9.32 loss:0.040 grdn:0.283 lr:4.6e-05 updt_s:0.811 data_s:1.706 smp/s:25 mem_gb:29.75
+Training:   7%|▋         | 2000/30000 [1:01:36<13:26:21,  1.73s/step]INFO 2026-07-23 03:33:23 ot_train.py:655 step:32K smpl:2M ep:46K epch:9.38 loss:0.039 grdn:0.285 lr:4.6e-05 updt_s:0.815 data_s:1.289 smp/s:30 mem_gb:29.76
+Training:   7%|▋         | 2200/30000 [1:08:01<13:29:15,  1.75s/step]INFO 2026-07-23 03:39:48 ot_train.py:655 step:32K smpl:2M ep:47K epch:9.44 loss:0.040 grdn:0.283 lr:4.5e-05 updt_s:0.846 data_s:1.077 smp/s:33 mem_gb:29.76
+Training:   8%|▊         | 2400/30000 [1:13:24<9:47:36,  1.28s/step]INFO 2026-07-23 03:45:11 ot_train.py:655 step:32K smpl:2M ep:47K epch:9.50 loss:0.040 grdn:0.282 lr:4.5e-05 updt_s:0.808 data_s:0.806 smp/s:40 mem_gb:29.76
+Training:   9%|▊         | 2600/30000 [1:18:08<9:50:49,  1.29s/step]INFO 2026-07-23 03:49:54 ot_train.py:655 step:33K smpl:2M ep:47K epch:9.55 loss:0.040 grdn:0.285 lr:4.4e-05 updt_s:0.858 data_s:0.559 smp/s:45 mem_gb:29.75
+Training:   9%|▉         | 2800/30000 [1:22:29<9:43:25,  1.29s/step]INFO 2026-07-23 03:54:16 ot_train.py:655 step:33K smpl:2M ep:47K epch:9.61 loss:0.039 grdn:0.279 lr:4.4e-05 updt_s:0.765 data_s:0.541 smp/s:49 mem_gb:29.75
+Training:  10%|█         | 3000/30000 [1:27:00<9:38:18,  1.29s/step]INFO 2026-07-23 03:58:47 ot_train.py:655 step:33K smpl:2M ep:48K epch:9.67 loss:0.039 grdn:0.291 lr:4.3e-05 updt_s:0.776 data_s:0.576 smp/s:47 mem_gb:29.76
+Training:  11%|█         | 3200/30000 [1:31:50<27:14:06,  3.66s/step]INFO 2026-07-23 04:03:37 ot_train.py:655 step:33K smpl:2M ep:48K epch:9.73 loss:0.039 grdn:0.285 lr:4.3e-05 updt_s:0.839 data_s:0.612 smp/s:44 mem_gb:29.75
+Training:  11%|█▏        | 3400/30000 [1:38:52<11:12:00,  1.52s/step]INFO 2026-07-23 04:10:39 ot_train.py:655 step:33K smpl:2M ep:48K epch:9.79 loss:0.039 grdn:0.285 lr:4.2e-05 updt_s:1.178 data_s:0.927 smp/s:30 mem_gb:29.75
+Training:  12%|█▏        | 3600/30000 [1:43:37<9:28:00,  1.29s/step]INFO 2026-07-23 04:15:23 ot_train.py:655 step:34K smpl:2M ep:49K epch:9.85 loss:0.038 grdn:0.272 lr:4.1e-05 updt_s:0.807 data_s:0.613 smp/s:45 mem_gb:29.75
+Training:  13%|█▎        | 3800/30000 [1:47:55<9:15:49,  1.27s/step]INFO 2026-07-23 04:19:42 ot_train.py:655 step:34K smpl:2M ep:49K epch:9.91 loss:0.038 grdn:0.288 lr:4.1e-05 updt_s:0.747 data_s:0.545 smp/s:50 mem_gb:29.75
+Training:  13%|█▎        | 4000/30000 [1:52:16<9:17:13,  1.29s/step]INFO 2026-07-23 04:24:02 ot_train.py:655 step:34K smpl:2M ep:49K epch:9.96 loss:0.038 grdn:0.283 lr:4.0e-05 updt_s:0.742 data_s:0.558 smp/s:49 mem_gb:29.74
+Training:  14%|█▍        | 4200/30000 [1:56:35<9:15:36,  1.29s/step]INFO 2026-07-23 04:28:22 ot_train.py:655 step:34K smpl:2M ep:49K epch:10.02 loss:0.037 grdn:0.284 lr:4.0e-05 updt_s:0.744 data_s:0.552 smp/s:49 mem_gb:29.76
+Training:  15%|█▍        | 4400/30000 [2:00:53<9:08:04,  1.28s/step]INFO 2026-07-23 04:32:40 ot_train.py:655 step:34K smpl:2M ep:50K epch:10.08 loss:0.037 grdn:0.290 lr:3.9e-05 updt_s:0.743 data_s:0.547 smp/s:50 mem_gb:29.75
+Training:  15%|█▌        | 4600/30000 [2:05:18<9:01:00,  1.28s/step]INFO 2026-07-23 04:37:05 ot_train.py:655 step:35K smpl:2M ep:50K epch:10.14 loss:0.038 grdn:0.301 lr:3.9e-05 updt_s:0.752 data_s:0.570 smp/s:48 mem_gb:29.76
+Training:  16%|█▌        | 4800/30000 [2:09:40<9:02:31,  1.29s/step]INFO 2026-07-23 04:41:26 ot_train.py:655 step:35K smpl:2M ep:50K epch:10.20 loss:0.037 grdn:0.295 lr:3.8e-05 updt_s:0.745 data_s:0.562 smp/s:49 mem_gb:29.75
+Training:  17%|█▋        | 5000/30000 [2:14:02<9:26:47,  1.36s/step]INFO 2026-07-23 04:45:48 ot_train.py:655 step:35K smpl:2M ep:51K epch:10.26 loss:0.037 grdn:0.288 lr:3.8e-05 updt_s:0.729 data_s:0.580 smp/s:49 mem_gb:29.76
+INFO 2026-07-23 04:45:48 ot_train.py:700 Checkpoint policy after step 35000
+INFO 2026-07-23 04:46:15 in_yaml.py:1413 Saved verified rolling checkpoint step=35000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00
+Training:  17%|█▋        | 5200/30000 [2:18:52<8:49:45,  1.28s/step]INFO 2026-07-23 04:50:39 ot_train.py:655 step:35K smpl:2M ep:51K epch:10.32 loss:0.037 grdn:0.289 lr:3.7e-05 updt_s:0.749 data_s:0.569 smp/s:49 mem_gb:29.75
+Training:  18%|█▊        | 5400/30000 [2:23:10<8:48:12,  1.29s/step]INFO 2026-07-23 04:54:57 ot_train.py:655 step:35K smpl:2M ep:51K epch:10.38 loss:0.036 grdn:0.288 lr:3.7e-05 updt_s:0.746 data_s:0.544 smp/s:50 mem_gb:29.75
+Training:  19%|█▊        | 5600/30000 [2:27:34<8:52:55,  1.31s/step]INFO 2026-07-23 04:59:20 ot_train.py:655 step:36K smpl:2M ep:51K epch:10.43 loss:0.037 grdn:0.292 lr:3.6e-05 updt_s:0.755 data_s:0.560 smp/s:49 mem_gb:29.75
+Training:  19%|█▉        | 5800/30000 [2:32:01<8:56:42,  1.33s/step]INFO 2026-07-23 05:03:48 ot_train.py:655 step:36K smpl:2M ep:52K epch:10.49 loss:0.036 grdn:0.293 lr:3.6e-05 updt_s:0.759 data_s:0.576 smp/s:48 mem_gb:29.76
+Training:  20%|██        | 6000/30000 [2:37:03<10:43:19,  1.61s/step]INFO 2026-07-23 05:08:50 ot_train.py:655 step:36K smpl:2M ep:52K epch:10.55 loss:0.035 grdn:0.289 lr:3.5e-05 updt_s:0.813 data_s:0.695 smp/s:42 mem_gb:29.76
+Training:  21%|██        | 6200/30000 [2:42:29<10:45:21,  1.63s/step]INFO 2026-07-23 05:14:16 ot_train.py:655 step:36K smpl:2M ep:52K epch:10.61 loss:0.035 grdn:0.276 lr:3.5e-05 updt_s:0.842 data_s:0.788 smp/s:39 mem_gb:29.74
+Training:  21%|██▏       | 6400/30000 [2:48:10<10:25:03,  1.59s/step]INFO 2026-07-23 05:19:57 ot_train.py:655 step:36K smpl:2M ep:53K epch:10.67 loss:0.034 grdn:0.284 lr:3.4e-05 updt_s:0.876 data_s:0.827 smp/s:38 mem_gb:29.76
+Training:  22%|██▏       | 6600/30000 [2:53:37<10:36:15,  1.63s/step]INFO 2026-07-23 05:25:24 ot_train.py:655 step:37K smpl:2M ep:53K epch:10.73 loss:0.033 grdn:0.270 lr:3.4e-05 updt_s:0.843 data_s:0.787 smp/s:39 mem_gb:29.75
+Training:  23%|██▎       | 6800/30000 [3:01:11<12:00:25,  1.86s/step]INFO 2026-07-23 05:32:58 ot_train.py:655 step:37K smpl:2M ep:53K epch:10.79 loss:0.033 grdn:0.284 lr:3.3e-05 updt_s:0.867 data_s:1.401 smp/s:28 mem_gb:29.76
+Training:  23%|██▎       | 7000/30000 [3:06:36<9:46:47,  1.53s/step] INFO 2026-07-23 05:38:23 ot_train.py:655 step:37K smpl:2M ep:53K epch:10.84 loss:0.034 grdn:0.302 lr:3.3e-05 updt_s:0.852 data_s:0.771 smp/s:39 mem_gb:29.75
+Training:  24%|██▍       | 7200/30000 [3:11:56<9:42:36,  1.53s/step]INFO 2026-07-23 05:43:42 ot_train.py:655 step:37K smpl:2M ep:54K epch:10.90 loss:0.035 grdn:0.291 lr:3.2e-05 updt_s:0.840 data_s:0.756 smp/s:40 mem_gb:29.76
+Training:  25%|██▍       | 7400/30000 [3:17:15<9:31:01,  1.52s/step]INFO 2026-07-23 05:49:02 ot_train.py:655 step:37K smpl:2M ep:54K epch:10.96 loss:0.035 grdn:0.289 lr:3.2e-05 updt_s:0.826 data_s:0.768 smp/s:40 mem_gb:29.76
+Training:  25%|██▌       | 7600/30000 [3:22:32<9:54:15,  1.59s/step] INFO 2026-07-23 05:54:19 ot_train.py:655 step:38K smpl:2M ep:54K epch:11.02 loss:0.034 grdn:0.296 lr:3.1e-05 updt_s:0.829 data_s:0.754 smp/s:40 mem_gb:29.75
+Training:  26%|██▌       | 7800/30000 [3:27:48<10:13:55,  1.66s/step]INFO 2026-07-23 05:59:35 ot_train.py:655 step:38K smpl:2M ep:55K epch:11.08 loss:0.033 grdn:0.298 lr:3.1e-05 updt_s:0.821 data_s:0.755 smp/s:41 mem_gb:29.76
+Training:  27%|██▋       | 8000/30000 [3:33:02<9:51:43,  1.61s/step]INFO 2026-07-23 06:04:49 ot_train.py:655 step:38K smpl:2M ep:55K epch:11.14 loss:0.034 grdn:0.286 lr:3.0e-05 updt_s:0.816 data_s:0.753 smp/s:41 mem_gb:29.75
+Training:  27%|██▋       | 8200/30000 [3:38:17<9:45:57,  1.61s/step]INFO 2026-07-23 06:10:04 ot_train.py:655 step:38K smpl:2M ep:55K epch:11.20 loss:0.033 grdn:0.287 lr:3.0e-05 updt_s:0.852 data_s:0.720 smp/s:41 mem_gb:29.75
+Training:  28%|██▊       | 8400/30000 [3:43:43<9:12:35,  1.53s/step]INFO 2026-07-23 06:15:30 ot_train.py:655 step:38K smpl:2M ep:55K epch:11.25 loss:0.034 grdn:0.293 lr:2.9e-05 updt_s:0.847 data_s:0.779 smp/s:39 mem_gb:29.74
+Training:  29%|██▊       | 8600/30000 [3:49:06<8:49:33,  1.48s/step]INFO 2026-07-23 06:20:52 ot_train.py:655 step:39K smpl:2M ep:56K epch:11.31 loss:0.031 grdn:0.287 lr:2.9e-05 updt_s:0.795 data_s:0.816 smp/s:40 mem_gb:29.75
+Training:  29%|██▉       | 8800/30000 [3:57:10<22:53:05,  3.89s/step]INFO 2026-07-23 06:28:57 ot_train.py:655 step:39K smpl:2M ep:56K epch:11.37 loss:0.031 grdn:0.300 lr:2.8e-05 updt_s:0.774 data_s:1.646 smp/s:26 mem_gb:29.75
+Training:  30%|███       | 9000/30000 [4:05:41<10:24:21,  1.78s/step]INFO 2026-07-23 06:37:28 ot_train.py:655 step:39K smpl:2M ep:56K epch:11.43 loss:0.032 grdn:0.295 lr:2.8e-05 updt_s:0.768 data_s:1.785 smp/s:25 mem_gb:29.76
+Training:  31%|███       | 9200/30000 [4:11:44<17:49:38,  3.09s/step]INFO 2026-07-23 06:43:31 ot_train.py:655 step:39K smpl:3M ep:57K epch:11.49 loss:0.031 grdn:0.285 lr:2.7e-05 updt_s:0.776 data_s:1.035 smp/s:35 mem_gb:29.76
+Training:  31%|███▏      | 9400/30000 [4:20:07<11:23:28,  1.99s/step]INFO 2026-07-23 06:51:53 ot_train.py:655 step:39K smpl:3M ep:57K epch:11.55 loss:0.031 grdn:0.280 lr:2.7e-05 updt_s:0.770 data_s:1.743 smp/s:25 mem_gb:29.76
+Training:  32%|███▏      | 9600/30000 [4:25:57<9:47:07,  1.73s/step]INFO 2026-07-23 06:57:43 ot_train.py:655 step:40K smpl:3M ep:57K epch:11.61 loss:0.031 grdn:0.307 lr:2.7e-05 updt_s:0.834 data_s:0.914 smp/s:37 mem_gb:29.75
+Training:  33%|███▎      | 9800/30000 [4:31:29<9:45:20,  1.74s/step]INFO 2026-07-23 07:03:16 ot_train.py:655 step:40K smpl:3M ep:58K epch:11.66 loss:0.032 grdn:0.291 lr:2.6e-05 updt_s:0.812 data_s:0.850 smp/s:39 mem_gb:29.75
+Training:  33%|███▎      | 10000/30000 [4:38:31<12:47:18,  2.30s/step]INFO 2026-07-23 07:10:18 ot_train.py:655 step:40K smpl:3M ep:58K epch:11.72 loss:0.031 grdn:0.303 lr:2.6e-05 updt_s:0.798 data_s:1.307 smp/s:30 mem_gb:29.76
+INFO 2026-07-23 07:10:18 ot_train.py:700 Checkpoint policy after step 40000
+INFO 2026-07-23 07:10:46 in_yaml.py:1413 Saved verified rolling checkpoint step=40000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01
+Training:  34%|███▍      | 10200/30000 [4:44:33<9:14:25,  1.68s/step]INFO 2026-07-23 07:16:20 ot_train.py:655 step:40K smpl:3M ep:58K epch:11.78 loss:0.030 grdn:0.300 lr:2.5e-05 updt_s:0.836 data_s:0.832 smp/s:38 mem_gb:29.75
+Training:  35%|███▍      | 10400/30000 [4:49:58<8:56:30,  1.64s/step]INFO 2026-07-23 07:21:44 ot_train.py:655 step:40K smpl:3M ep:58K epch:11.84 loss:0.030 grdn:0.291 lr:2.5e-05 updt_s:0.817 data_s:0.802 smp/s:40 mem_gb:29.75
+Training:  35%|███▌      | 10600/30000 [4:55:16<8:13:51,  1.53s/step]INFO 2026-07-23 07:27:02 ot_train.py:655 step:41K smpl:3M ep:59K epch:11.90 loss:0.030 grdn:0.282 lr:2.4e-05 updt_s:0.808 data_s:0.780 smp/s:40 mem_gb:29.76
+Training:  36%|███▌      | 10800/30000 [5:00:58<8:41:31,  1.63s/step]INFO 2026-07-23 07:32:45 ot_train.py:655 step:41K smpl:3M ep:59K epch:11.96 loss:0.029 grdn:0.277 lr:2.4e-05 updt_s:0.816 data_s:0.896 smp/s:37 mem_gb:29.74
+Training:  37%|███▋      | 11000/30000 [5:06:39<8:44:25,  1.66s/step]INFO 2026-07-23 07:38:25 ot_train.py:655 step:41K smpl:3M ep:59K epch:12.02 loss:0.029 grdn:0.283 lr:2.3e-05 updt_s:0.870 data_s:0.830 smp/s:38 mem_gb:29.76
+Training:  37%|███▋      | 11200/30000 [5:12:30<8:22:54,  1.61s/step]INFO 2026-07-23 07:44:17 ot_train.py:655 step:41K smpl:3M ep:60K epch:12.08 loss:0.029 grdn:0.283 lr:2.3e-05 updt_s:0.851 data_s:0.903 smp/s:36 mem_gb:29.75
+Training:  38%|███▊      | 11400/30000 [5:19:14<9:55:31,  1.92s/step] INFO 2026-07-23 07:51:01 ot_train.py:655 step:41K smpl:3M ep:60K epch:12.13 loss:0.029 grdn:0.286 lr:2.2e-05 updt_s:0.816 data_s:1.204 smp/s:32 mem_gb:29.75
+Training:  39%|███▊      | 11600/30000 [5:26:43<8:02:20,  1.57s/step]INFO 2026-07-23 07:58:29 ot_train.py:655 step:42K smpl:3M ep:60K epch:12.19 loss:0.028 grdn:0.268 lr:2.2e-05 updt_s:0.773 data_s:1.466 smp/s:29 mem_gb:29.76
+Training:  39%|███▉      | 11800/30000 [5:33:22<13:52:52,  2.75s/step]INFO 2026-07-23 08:05:08 ot_train.py:655 step:42K smpl:3M ep:60K epch:12.25 loss:0.029 grdn:0.288 lr:2.2e-05 updt_s:0.800 data_s:1.192 smp/s:32 mem_gb:29.76
+Training:  40%|████      | 12000/30000 [5:38:54<7:59:20,  1.60s/step]INFO 2026-07-23 08:10:41 ot_train.py:655 step:42K smpl:3M ep:61K epch:12.31 loss:0.028 grdn:0.285 lr:2.1e-05 updt_s:0.812 data_s:0.848 smp/s:39 mem_gb:29.76
+Training:  41%|████      | 12200/30000 [5:45:05<8:07:10,  1.64s/step]INFO 2026-07-23 08:16:52 ot_train.py:655 step:42K smpl:3M ep:61K epch:12.37 loss:0.028 grdn:0.292 lr:2.1e-05 updt_s:0.817 data_s:1.038 smp/s:34 mem_gb:29.75
+Training:  41%|████▏     | 12400/30000 [5:50:32<7:37:26,  1.56s/step]INFO 2026-07-23 08:22:19 ot_train.py:655 step:42K smpl:3M ep:61K epch:12.43 loss:0.028 grdn:0.286 lr:2.0e-05 updt_s:0.834 data_s:0.798 smp/s:39 mem_gb:29.75
+Training:  42%|████▏     | 12600/30000 [5:58:52<13:00:55,  2.69s/step]INFO 2026-07-23 08:30:39 ot_train.py:655 step:43K smpl:3M ep:62K epch:12.49 loss:0.028 grdn:0.272 lr:2.0e-05 updt_s:0.769 data_s:1.728 smp/s:26 mem_gb:29.75
+Training:  43%|████▎     | 12800/30000 [6:10:08<11:39:51,  2.44s/step]INFO 2026-07-23 08:41:55 ot_train.py:655 step:43K smpl:3M ep:62K epch:12.54 loss:0.028 grdn:0.286 lr:1.9e-05 updt_s:0.833 data_s:2.548 smp/s:19 mem_gb:29.75
+Training:  43%|████▎     | 13000/30000 [6:15:41<7:26:17,  1.58s/step]INFO 2026-07-23 08:47:28 ot_train.py:655 step:43K smpl:3M ep:62K epch:12.60 loss:0.027 grdn:0.288 lr:1.9e-05 updt_s:0.800 data_s:0.863 smp/s:38 mem_gb:29.74
+Training:  44%|████▍     | 13200/30000 [6:21:12<7:13:46,  1.55s/step]INFO 2026-07-23 08:52:58 ot_train.py:655 step:43K smpl:3M ep:62K epch:12.66 loss:0.026 grdn:0.275 lr:1.9e-05 updt_s:0.814 data_s:0.835 smp/s:39 mem_gb:29.76
+Training:  45%|████▍     | 13400/30000 [6:26:37<7:53:06,  1.71s/step]INFO 2026-07-23 08:58:24 ot_train.py:655 step:43K smpl:3M ep:63K epch:12.72 loss:0.026 grdn:0.279 lr:1.8e-05 updt_s:0.816 data_s:0.808 smp/s:39 mem_gb:29.76
+Training:  45%|████▌     | 13600/30000 [6:32:00<6:59:39,  1.54s/step]INFO 2026-07-23 09:03:47 ot_train.py:655 step:44K smpl:3M ep:63K epch:12.78 loss:0.026 grdn:0.267 lr:1.8e-05 updt_s:0.794 data_s:0.821 smp/s:40 mem_gb:29.75
+Training:  46%|████▌     | 13800/30000 [6:38:28<10:20:43,  2.30s/step]INFO 2026-07-23 09:10:15 ot_train.py:655 step:44K smpl:3M ep:63K epch:12.84 loss:0.027 grdn:0.269 lr:1.7e-05 updt_s:0.766 data_s:1.174 smp/s:33 mem_gb:29.75
+Training:  47%|████▋     | 14000/30000 [6:44:07<6:51:38,  1.54s/step]INFO 2026-07-23 09:15:53 ot_train.py:655 step:44K smpl:3M ep:64K epch:12.90 loss:0.026 grdn:0.270 lr:1.7e-05 updt_s:0.816 data_s:0.873 smp/s:38 mem_gb:29.75
+Training:  47%|████▋     | 14200/30000 [6:49:58<7:53:26,  1.80s/step]INFO 2026-07-23 09:21:45 ot_train.py:655 step:44K smpl:3M ep:64K epch:12.95 loss:0.026 grdn:0.285 lr:1.7e-05 updt_s:0.828 data_s:0.928 smp/s:36 mem_gb:29.75
+Training:  48%|████▊     | 14400/30000 [6:57:32<6:58:09,  1.61s/step]INFO 2026-07-23 09:29:18 ot_train.py:655 step:44K smpl:3M ep:64K epch:13.01 loss:0.026 grdn:0.271 lr:1.6e-05 updt_s:0.781 data_s:1.484 smp/s:28 mem_gb:29.76
+Training:  49%|████▊     | 14600/30000 [7:02:51<6:43:15,  1.57s/step]INFO 2026-07-23 09:34:37 ot_train.py:655 step:45K smpl:3M ep:64K epch:13.07 loss:0.026 grdn:0.271 lr:1.6e-05 updt_s:0.819 data_s:0.774 smp/s:40 mem_gb:29.75
+Training:  49%|████▉     | 14800/30000 [7:08:09<6:45:50,  1.60s/step]INFO 2026-07-23 09:39:56 ot_train.py:655 step:45K smpl:3M ep:65K epch:13.13 loss:0.025 grdn:0.278 lr:1.5e-05 updt_s:0.811 data_s:0.779 smp/s:40 mem_gb:29.75
+Training:  50%|█████     | 15000/30000 [7:13:36<6:48:40,  1.63s/step]INFO 2026-07-23 09:45:22 ot_train.py:655 step:45K smpl:3M ep:65K epch:13.19 loss:0.026 grdn:0.285 lr:1.5e-05 updt_s:0.823 data_s:0.808 smp/s:39 mem_gb:29.76
+INFO 2026-07-23 09:45:22 ot_train.py:700 Checkpoint policy after step 45000
+INFO 2026-07-23 09:45:51 in_yaml.py:1413 Saved verified rolling checkpoint step=45000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00
+Training:  51%|█████     | 15200/30000 [7:19:28<6:39:25,  1.62s/step]INFO 2026-07-23 09:51:14 ot_train.py:655 step:45K smpl:3M ep:65K epch:13.25 loss:0.025 grdn:0.279 lr:1.5e-05 updt_s:0.832 data_s:0.785 smp/s:40 mem_gb:29.74
+Training:  51%|█████▏    | 15400/30000 [7:24:55<6:21:42,  1.57s/step]INFO 2026-07-23 09:56:42 ot_train.py:655 step:45K smpl:3M ep:66K epch:13.31 loss:0.025 grdn:0.276 lr:1.4e-05 updt_s:0.845 data_s:0.790 smp/s:39 mem_gb:29.75
+Training:  52%|█████▏    | 15600/30000 [7:30:14<6:14:31,  1.56s/step]INFO 2026-07-23 10:02:00 ot_train.py:655 step:46K smpl:3M ep:66K epch:13.36 loss:0.025 grdn:0.273 lr:1.4e-05 updt_s:0.815 data_s:0.775 smp/s:40 mem_gb:29.75
+Training:  53%|█████▎    | 15800/30000 [7:35:46<8:09:00,  2.07s/step]INFO 2026-07-23 10:07:33 ot_train.py:655 step:46K smpl:3M ep:66K epch:13.42 loss:0.024 grdn:0.271 lr:1.4e-05 updt_s:0.812 data_s:0.849 smp/s:39 mem_gb:29.75
+Training:  53%|█████▎    | 16000/30000 [7:42:16<7:18:23,  1.88s/step]INFO 2026-07-23 10:14:03 ot_train.py:655 step:46K smpl:3M ep:66K epch:13.48 loss:0.025 grdn:0.278 lr:1.3e-05 updt_s:0.830 data_s:1.117 smp/s:33 mem_gb:29.75
+Training:  54%|█████▍    | 16200/30000 [7:48:31<6:19:45,  1.65s/step]INFO 2026-07-23 10:20:18 ot_train.py:655 step:46K smpl:3M ep:67K epch:13.54 loss:0.024 grdn:0.271 lr:1.3e-05 updt_s:0.844 data_s:1.031 smp/s:34 mem_gb:29.76
+Training:  55%|█████▍    | 16400/30000 [7:55:57<6:43:51,  1.78s/step]INFO 2026-07-23 10:27:44 ot_train.py:655 step:46K smpl:3M ep:67K epch:13.60 loss:0.024 grdn:0.264 lr:1.3e-05 updt_s:0.849 data_s:1.380 smp/s:29 mem_gb:29.76
+Training:  55%|█████▌    | 16600/30000 [8:03:15<8:29:49,  2.28s/step]INFO 2026-07-23 10:35:02 ot_train.py:655 step:47K smpl:3M ep:67K epch:13.66 loss:0.024 grdn:0.266 lr:1.2e-05 updt_s:0.832 data_s:1.357 smp/s:29 mem_gb:29.76
+Training:  56%|█████▌    | 16800/30000 [8:09:00<5:24:35,  1.48s/step]INFO 2026-07-23 10:40:46 ot_train.py:655 step:47K smpl:3M ep:68K epch:13.72 loss:0.024 grdn:0.270 lr:1.2e-05 updt_s:0.846 data_s:0.873 smp/s:37 mem_gb:29.75
+Training:  57%|█████▋    | 17000/30000 [8:16:03<13:47:13,  3.82s/step]INFO 2026-07-23 10:47:50 ot_train.py:655 step:47K smpl:3M ep:68K epch:13.77 loss:0.024 grdn:0.280 lr:1.1e-05 updt_s:1.418 data_s:0.699 smp/s:30 mem_gb:29.75
+Training:  57%|█████▋    | 17200/30000 [8:30:06<12:59:51,  3.66s/step]INFO 2026-07-23 11:01:53 ot_train.py:655 step:47K smpl:3M ep:68K epch:13.83 loss:0.025 grdn:0.277 lr:1.1e-05 updt_s:3.257 data_s:0.956 smp/s:15 mem_gb:29.76
+Training:  58%|█████▊    | 17400/30000 [8:44:15<14:41:51,  4.20s/step]INFO 2026-07-23 11:16:02 ot_train.py:655 step:47K smpl:3M ep:68K epch:13.89 loss:0.024 grdn:0.277 lr:1.1e-05 updt_s:3.363 data_s:0.879 smp/s:15 mem_gb:29.74
+Training:  59%|█████▊    | 17600/30000 [8:58:28<16:21:34,  4.75s/step]INFO 2026-07-23 11:30:15 ot_train.py:655 step:48K smpl:3M ep:69K epch:13.95 loss:0.023 grdn:0.250 lr:1.0e-05 updt_s:3.249 data_s:1.014 smp/s:15 mem_gb:29.76
+Training:  59%|█████▉    | 17800/30000 [9:13:51<12:53:10,  3.80s/step]INFO 2026-07-23 11:45:38 ot_train.py:655 step:48K smpl:3M ep:69K epch:14.01 loss:0.023 grdn:0.259 lr:1.0e-05 updt_s:3.156 data_s:1.459 smp/s:14 mem_gb:29.76
+Training:  60%|██████    | 18000/30000 [9:27:47<14:39:06,  4.40s/step]INFO 2026-07-23 11:59:33 ot_train.py:655 step:48K smpl:3M ep:69K epch:14.07 loss:0.023 grdn:0.268 lr:9.9e-06 updt_s:3.340 data_s:0.835 smp/s:15 mem_gb:29.75
+Training:  61%|██████    | 18200/30000 [9:42:00<13:21:08,  4.07s/step]INFO 2026-07-23 12:13:46 ot_train.py:655 step:48K smpl:3M ep:70K epch:14.13 loss:0.023 grdn:0.256 lr:9.5e-06 updt_s:3.341 data_s:0.921 smp/s:15 mem_gb:29.76
+Training:  61%|██████▏   | 18400/30000 [9:55:45<11:23:51,  3.54s/step]INFO 2026-07-23 12:27:32 ot_train.py:655 step:48K smpl:3M ep:70K epch:14.19 loss:0.023 grdn:0.263 lr:9.2e-06 updt_s:3.395 data_s:0.730 smp/s:16 mem_gb:29.76
+Training:  62%|██████▏   | 18600/30000 [10:09:20<13:26:35,  4.25s/step]INFO 2026-07-23 12:41:06 ot_train.py:655 step:49K smpl:3M ep:70K epch:14.24 loss:0.023 grdn:0.263 lr:8.9e-06 updt_s:3.409 data_s:0.663 smp/s:16 mem_gb:29.75
+Training:  63%|██████▎   | 18800/30000 [10:22:36<12:10:59,  3.92s/step]INFO 2026-07-23 12:54:23 ot_train.py:655 step:49K smpl:3M ep:71K epch:14.30 loss:0.023 grdn:0.273 lr:8.6e-06 updt_s:3.333 data_s:0.649 smp/s:16 mem_gb:29.75
+Training:  63%|██████▎   | 19000/30000 [10:36:12<11:44:53,  3.84s/step]INFO 2026-07-23 13:07:59 ot_train.py:655 step:49K smpl:3M ep:71K epch:14.36 loss:0.022 grdn:0.249 lr:8.3e-06 updt_s:3.426 data_s:0.651 smp/s:16 mem_gb:29.75
+Training:  64%|██████▍   | 19200/30000 [10:49:40<11:44:05,  3.91s/step]INFO 2026-07-23 13:21:26 ot_train.py:655 step:49K smpl:3M ep:71K epch:14.42 loss:0.023 grdn:0.268 lr:8.1e-06 updt_s:3.438 data_s:0.599 smp/s:16 mem_gb:29.75
+Training:  65%|██████▍   | 19400/30000 [11:03:29<11:36:09,  3.94s/step]INFO 2026-07-23 13:35:16 ot_train.py:655 step:49K smpl:3M ep:71K epch:14.48 loss:0.023 grdn:0.263 lr:7.8e-06 updt_s:3.371 data_s:0.774 smp/s:15 mem_gb:29.75
+Training:  65%|██████▌   | 19600/30000 [11:17:18<12:37:29,  4.37s/step]INFO 2026-07-23 13:49:05 ot_train.py:655 step:50K smpl:3M ep:72K epch:14.54 loss:0.022 grdn:0.260 lr:7.5e-06 updt_s:3.439 data_s:0.706 smp/s:15 mem_gb:29.76
+Training:  66%|██████▌   | 19800/30000 [11:30:50<10:47:22,  3.81s/step]INFO 2026-07-23 14:02:37 ot_train.py:655 step:50K smpl:3M ep:72K epch:14.60 loss:0.021 grdn:0.246 lr:7.2e-06 updt_s:3.478 data_s:0.578 smp/s:16 mem_gb:29.74
+Training:  67%|██████▋   | 20000/30000 [11:44:28<11:22:23,  4.09s/step]INFO 2026-07-23 14:16:14 ot_train.py:655 step:50K smpl:3M ep:72K epch:14.65 loss:0.021 grdn:0.253 lr:6.9e-06 updt_s:3.486 data_s:0.600 smp/s:16 mem_gb:29.75
+INFO 2026-07-23 14:16:14 ot_train.py:700 Checkpoint policy after step 50000
+INFO 2026-07-23 14:16:45 in_yaml.py:1413 Saved verified rolling checkpoint step=50000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01
+Training:  67%|██████▋   | 20083/30000 [11:50:32<10:38:12,  3.86s/step]

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/files/requirements.txt

@@ -0,0 +1,117 @@
+tzdata==2026.3
+nvidia-cufile-cu12==1.13.1.3
+wandb==0.27.2
+six==1.17.0
+peft==0.19.1
+nvidia-curand-cu12==10.3.9.90
+nvidia-cuda-runtime-cu12==12.8.90
+datasets==4.8.5
+attrs==26.1.0
+cloudpickle==3.1.2
+packaging==25.0
+nvidia-cublas-cu12==12.8.4.1
+cmake==4.1.3
+psutil==7.2.2
+pydantic==2.13.4
+GitPython==3.1.51
+pillow==12.2.0
+pytz==2026.2
+cuda-pathfinder==1.2.2
+nvidia-cusparse-cu12==12.5.8.93
+typing_extensions==4.15.0
+urllib3==2.7.0
+pydantic_core==2.46.4
+draccus==0.10.0
+mypy_extensions==1.1.0
+dill==0.4.1
+typer==0.26.8
+decord==0.6.0
+tqdm==4.68.4
+triton==3.6.0
+idna==3.18
+requests==2.34.2
+nvidia-cudnn-cu12==9.19.0.56
+mdurl==0.1.2
+shellingham==1.5.4
+zipp==4.1.0
+timm==1.0.28
+torch==2.11.0+cu128
+tokenizers==0.22.2
+nvidia-cusparselt-cu12==0.7.1
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cusolver-cu12==11.7.3.90
+opencv-python-headless==4.13.0.92
+httpx==0.28.1
+Jinja2==3.1.6
+einops==0.8.2
+mergedeep==1.3.4
+platformdirs==4.10.0
+nvidia-nvjitlink-cu12==12.8.93
+lerobot==0.6.0
+PyYAML==6.0.3
+accelerate==1.14.0
+anyio==4.14.2
+gymnasium==1.3.0
+pip==26.1.2
+huggingface_hub==1.23.0
+diffusers==0.35.2
+annotated-doc==0.0.4
+pyarrow==25.0.0
+python-dateutil==2.9.0.post0
+termcolor==3.3.0
+nvidia-nccl-cu12==2.28.9
+sentry-sdk==2.65.0
+torchcodec==0.11.1
+pyyaml-include==1.4.1
+multiprocess==0.70.19
+wrapt==2.2.2
+typing-inspection==0.4.2
+nvidia-cuda-nvrtc-cu12==12.8.93
+Pygments==2.20.0
+nvidia-cuda-cupti-cu12==12.8.90
+certifi==2026.6.17
+setuptools==78.1.0
+charset-normalizer==3.4.9
+propcache==0.5.2
+multidict==6.7.1
+regex==2026.7.10
+frozenlist==1.8.0
+smmap==5.0.3
+hf-xet==1.5.1
+torchvision==0.26.0+cu128
+gitdb==4.0.12
+transformers==5.5.4
+rich==15.0.0
+cuda-bindings==12.9.4
+protobuf==7.35.1
+aiosignal==1.4.0
+jsonlines==4.0.0
+yarl==1.24.2
+filelock==3.29.0
+av==15.1.0
+networkx==3.6.1
+sympy==1.14.0
+wheel==0.47.0
+nvidia-nvtx-cu12==12.8.90
+xxhash==3.8.1
+toml==0.10.2
+MarkupSafe==3.0.3
+pandas==2.3.3
+safetensors==0.8.0
+click==8.4.2
+fsspec==2026.2.0
+dm-tree==0.1.10
+absl-py==2.5.0
+Farama-Notifications==0.0.6
+mpmath==1.3.0
+importlib_metadata==9.0.0
+markdown-it-py==4.2.0
+h11==0.16.0
+typing-inspect==0.9.0
+httpcore==1.0.9
+annotated-types==0.7.0
+numpy==2.2.6
+nvidia-nvshmem-cu12==3.4.5
+aiohappyeyeballs==2.7.1
+cuda-toolkit==12.8.1
+aiohttp==3.14.1

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/files/wandb-metadata.json

@@ -0,0 +1,53 @@
+{
+  "os": "Linux-6.8.0-55-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-23T02:31:24.518815Z",
+  "args": [
+    "--config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model/train_config.json",
+    "--resume=true",
+    "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline",
+    "--dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
+    "--wandb.run_id=jeqgwsec"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePathLocal": "my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a329728c2c9524d79b966c4a7a82ee56a4c2abf7"
+  },
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline",
+  "host": "ext_minje-box",
+  "executable": "/home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 2,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "1138103230464"
+    }
+  },
+  "memory": {
+    "total": "2164176568320"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "nroouvr28mdugxx2qqhtz4l7z549sop5"
+}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2026-07-23T02:31:24.554092789Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4mx05xmn/port-2500948.txt","pid":2500948,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T02:31:24.555020572Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2500948}
+{"time":"2026-07-23T02:31:24.554911221Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2500948-2501761-571158799/socket","Net":"unix"}}
+{"time":"2026-07-23T02:31:24.737398925Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T02:31:24.744768718Z","level":"INFO","msg":"handleInformInit: received","streamId":"jeqgwsec","id":"1(@)"}
+{"time":"2026-07-23T02:31:25.126628253Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"jeqgwsec","id":"1(@)"}
+{"time":"2026-07-23T02:31:30.905568957Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ibi2eue7z1ot"}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-23 02:31:24,519 INFO    MainThread:2500948 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-23 02:31:24,519 INFO    MainThread:2500948 [wandb_setup.py:_flush():81] Configure stats pid to 2500948
+2026-07-23 02:31:24,519 INFO    MainThread:2500948 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-23 02:31:24,520 INFO    MainThread:2500948 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/logs/debug.log
+2026-07-23 02:31:24,520 INFO    MainThread:2500948 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/logs/debug-internal.log
+2026-07-23 02:31:24,520 INFO    MainThread:2500948 [wandb_init.py:init():766] calling init triggers
+2026-07-23 02:31:24,520 INFO    MainThread:2500948 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'jeqgwsec', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000', '_wandb': {}}
+2026-07-23 02:31:24,520 INFO    MainThread:2500948 [wandb_init.py:init():814] starting backend
+2026-07-23 02:31:24,737 INFO    MainThread:2500948 [wandb_init.py:init():829] sending inform_init request
+2026-07-23 02:31:25,127 INFO    MainThread:2500948 [wandb_init.py:init():834] backend started and connected
+2026-07-23 02:31:25,130 INFO    MainThread:2500948 [wandb_init.py:init():904] updated telemetry
+2026-07-23 02:31:25,134 INFO    MainThread:2500948 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-23 02:31:25,832 INFO    MainThread:2500948 [wandb_init.py:init():972] starting run threads in backend
+2026-07-23 02:31:25,900 INFO    MainThread:2500948 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-23 02:31:25,900 INFO    MainThread:2500948 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-23 02:31:25,900 INFO    MainThread:2500948 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-23 02:31:25,900 INFO    MainThread:2500948 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-23 02:31:25,902 INFO    MainThread:2500948 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/config.yaml

@@ -0,0 +1,329 @@
+_wandb:
+    value:
+        cli_version: 0.27.2
+        e:
+            q52i38bd1zs4dxfa7y10j3r5h3dxa68o:
+                args:
+                    - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/train_config.json
+                    - --resume=true
+                codePath: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                codePathLocal: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.9"
+                disk:
+                    /:
+                        total: "1888425144320"
+                        used: "1327086579712"
+                executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python
+                gpu: NVIDIA H200
+                gpu_count: 2
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "150754820096"
+                      name: NVIDIA H200
+                      uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc
+                host: ext_minje-box
+                memory:
+                    total: "2164176568320"
+                os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39
+                program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
+                python: CPython 3.12.13
+                root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline
+                startedAt: "2026-07-23T16:35:01.611488Z"
+                writerId: q52i38bd1zs4dxfa7y10j3r5h3dxa68o
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.12.13
+            "5": 0.27.2
+            "6": 5.5.4
+            "10":
+                - 21
+            "12": 0.27.2
+            "13": linux-x86_64
+batch_size:
+    value: 64
+checkpoint_path:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01
+cudnn_deterministic:
+    value: false
+dataset:
+    value:
+        depth_output_unit: mm
+        episodes: null
+        eval_split: 0
+        image_transforms:
+            enable: false
+            max_num_transforms: 3
+            random_order: false
+            tfs:
+                affine:
+                    kwargs:
+                        degrees:
+                            - -5
+                            - 5
+                        translate:
+                            - 0.05
+                            - 0.05
+                    type: RandomAffine
+                    weight: 1
+                brightness:
+                    kwargs:
+                        brightness:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                contrast:
+                    kwargs:
+                        contrast:
+                            - 0.8
+                            - 1.2
+                    type: ColorJitter
+                    weight: 1
+                hue:
+                    kwargs:
+                        hue:
+                            - -0.05
+                            - 0.05
+                    type: ColorJitter
+                    weight: 1
+                saturation:
+                    kwargs:
+                        saturation:
+                            - 0.5
+                            - 1.5
+                    type: ColorJitter
+                    weight: 1
+                sharpness:
+                    kwargs:
+                        sharpness:
+                            - 0.5
+                            - 1.5
+                    type: SharpnessJitter
+                    weight: 1
+        repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+        return_uint8: false
+        revision: null
+        root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+        streaming: false
+        use_imagenet_stats: true
+        video_backend: torchcodec
+env:
+    value: null
+env_eval_freq:
+    value: 20000
+eval:
+    value:
+        batch_size: 50
+        n_episodes: 50
+        recording: false
+        recording_private: false
+        recording_repo_id: null
+        use_async_envs: true
+eval_steps:
+    value: 0
+job:
+    value:
+        detach: false
+        image: huggingface/lerobot-gpu:latest
+        tags: []
+        target: null
+        timeout: 2d
+job_name:
+    value: INSIGHT_6D_DiT16_Baseline
+log_freq:
+    value: 200
+max_eval_samples:
+    value: 0
+num_workers:
+    value: 4
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.999
+        eps: 1e-08
+        grad_clip_norm: 1
+        lr: 0.0001
+        type: adamw
+        weight_decay: 1e-05
+output_dir:
+    value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline
+peft:
+    value: null
+persistent_workers:
+    value: true
+policy:
+    value:
+        action_decode_transform: null
+        action_expert_num_layers: 16
+        balance_dataset_weights: true
+        balance_trajectory_weights: true
+        base_model_path: nvidia/GR00T-N1.7-3B
+        batch_size: 32
+        chunk_size: 16
+        dataloader_num_workers: 8
+        dataset_paths: null
+        device: cuda
+        embodiment_tag: new_embodiment
+        image_size:
+            - 256
+            - 256
+        input_features:
+            observation.images.guide:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.right_shoulder:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.images.wrist:
+                shape:
+                    - 3
+                    - 224
+                    - 224
+                type: VISUAL
+            observation.state:
+                shape:
+                    - 16
+                type: STATE
+        license: null
+        lora_alpha: 16
+        lora_dropout: 0.1
+        lora_full_model: false
+        lora_rank: 0
+        max_action_dim: 132
+        max_state_dim: 132
+        max_steps: 10000
+        model_params_fp32: true
+        n_action_steps: 16
+        n_obs_steps: 1
+        normalization_mapping:
+            ACTION: IDENTITY
+            STATE: IDENTITY
+            VISUAL: IDENTITY
+        num_inference_timesteps: null
+        optimizer_betas:
+            - 0.9
+            - 0.999
+        optimizer_eps: 1e-08
+        optimizer_lr: 0.0001
+        optimizer_weight_decay: 1e-05
+        output_dir: ./tmp/gr00t
+        output_features:
+            action:
+                shape:
+                    - 10
+                type: ACTION
+        pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model
+        pretrained_revision: null
+        private: null
+        push_to_hub: false
+        relative_exclude_joints: []
+        repo_id: null
+        report_to: wandb
+        resume: false
+        rkd_angle_weight: 1
+        rkd_distance_weight: 1
+        rkd_enabled: false
+        rkd_fm_loss_weight: 1
+        rkd_loss_weight: 0.1
+        rkd_student_attention_hidden_dim: 512
+        rkd_student_attention_output_dim: 512
+        rkd_student_pooling: attention
+        rkd_vae_checkpoint: null
+        rtc_ramp_rate: null
+        save_steps: 1000
+        tags: null
+        tokenizer_assets_repo: null
+        tune_diffusion_model: true
+        tune_llm: false
+        tune_projector: true
+        tune_top_llm_layers: 0
+        tune_visual: false
+        tune_vlln: true
+        type: groot_rkd
+        use_amp: false
+        use_bf16: true
+        use_flash_attention: false
+        use_peft: false
+        use_relative_actions: false
+        video_backend: decord
+        warmup_ratio: 0.05
+        warmup_steps: null
+prefetch_factor:
+    value: 4
+resume:
+    value: true
+reward_model:
+    value: null
+sample_weighting:
+    value: null
+save_checkpoint:
+    value: true
+save_checkpoint_to_hub:
+    value: false
+save_freq:
+    value: 20000
+scheduler:
+    value:
+        name: cosine
+        num_warmup_steps: 500
+        type: diffuser
+seed:
+    value: 42
+steps:
+    value: 60000
+tolerance_s:
+    value: 0.0001
+use_policy_training_preset:
+    value: true
+wandb:
+    value:
+        add_tags: true
+        disable_artifact: true
+        enable: true
+        entity: null
+        mode: null
+        notes: null
+        project: lerobot
+        run_id: jeqgwsec

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/output.log

@@ -0,0 +1,96 @@
+INFO 2026-07-23 16:35:04 db_utils.py:126 Logs will be synced with wandb.
+INFO 2026-07-23 16:35:04 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/jeqgwsec
+INFO 2026-07-23 16:35:04 ot_train.py:310 Creating dataset
+INFO 2026-07-23 16:35:06 ot_train.py:344 Creating policy
+INFO 2026-07-23 16:35:06 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model
+INFO 2026-07-23 16:35:06 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 332.64it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 807/807 [00:01<00:00, 424.83it/s] 
+GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495
+Key                                                                  | Status     |  |
+---------------------------------------------------------------------+------------+--+-
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.bias    | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.weight   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.2.weight       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_v.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_q.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.2.bias         | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.bias       | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.bias   | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.weight  | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.weight | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.attn1.to_k.weight     | UNEXPECTED |  |
+action_head.model.transformer_blocks.{16...31}.norm1.linear.bias     | UNEXPECTED |  |
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+Loading weights from local directory
+INFO 2026-07-23 16:35:28 ot_train.py:417 Creating optimizer and scheduler
+INFO 2026-07-23 16:35:31 ot_train.py:449 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline
+INFO 2026-07-23 16:35:31 ot_train.py:456 cfg.steps=60000 (60K)
+INFO 2026-07-23 16:35:31 ot_train.py:457 dataset.num_frames=218367 (218K)
+INFO 2026-07-23 16:35:31 ot_train.py:458 dataset.num_episodes=4930
+INFO 2026-07-23 16:35:31 ot_train.py:461 Effective batch size: 64 x 1 = 64
+INFO 2026-07-23 16:35:31 ot_train.py:462 num_learnable_params=1079180416 (1B)
+INFO 2026-07-23 16:35:31 ot_train.py:463 num_total_params=2604255361 (3B)
+INFO 2026-07-23 16:35:31 ot_train.py:504 Resuming data order at epoch 22, sample 22144
+Training:   0%|          | 0/10000 [00:00<?, ?step/s]INFO 2026-07-23 16:35:32 ot_train.py:614 Start offline training on a fixed dataset, with effective batch size: 64
+Training:   2%|▏         | 200/10000 [13:50<10:59:17,  4.04s/step]INFO 2026-07-23 16:49:23 ot_train.py:658 step:50K smpl:3M ep:73K epch:14.71 loss:0.022 grdn:0.264 lr:6.7e-06 updt_s:3.599 data_s:0.554 smp/s:15 mem_gb:29.76
+Training:   4%|▍         | 400/10000 [27:29<10:35:46,  3.97s/step]INFO 2026-07-23 17:03:02 ot_train.py:658 step:50K smpl:3M ep:73K epch:14.77 loss:0.021 grdn:0.251 lr:6.4e-06 updt_s:3.563 data_s:0.531 smp/s:16 mem_gb:29.76
+Training:   6%|▌         | 600/10000 [41:05<10:12:26,  3.91s/step]INFO 2026-07-23 17:16:38 ot_train.py:658 step:51K smpl:3M ep:73K epch:14.83 loss:0.021 grdn:0.253 lr:6.2e-06 updt_s:3.544 data_s:0.532 smp/s:16 mem_gb:29.76
+Training:   8%|▊         | 800/10000 [54:40<10:17:50,  4.03s/step]INFO 2026-07-23 17:30:13 ot_train.py:658 step:51K smpl:3M ep:73K epch:14.89 loss:0.021 grdn:0.254 lr:5.9e-06 updt_s:3.545 data_s:0.532 smp/s:16 mem_gb:29.76
+Training:  10%|█         | 1000/10000 [1:08:15<10:03:21,  4.02s/step]INFO 2026-07-23 17:43:48 ot_train.py:658 step:51K smpl:3M ep:74K epch:14.95 loss:0.021 grdn:0.256 lr:5.7e-06 updt_s:3.542 data_s:0.531 smp/s:16 mem_gb:29.76
+Training:  12%|█▏        | 1200/10000 [1:21:51<10:06:06,  4.13s/step]INFO 2026-07-23 17:57:24 ot_train.py:658 step:51K smpl:3M ep:74K epch:15.01 loss:0.021 grdn:0.257 lr:5.4e-06 updt_s:3.545 data_s:0.533 smp/s:16 mem_gb:29.76
+Training:  14%|█▍        | 1400/10000 [1:35:29<9:34:59,  4.01s/step] INFO 2026-07-23 18:11:02 ot_train.py:658 step:51K smpl:3M ep:74K epch:15.06 loss:0.021 grdn:0.257 lr:5.2e-06 updt_s:3.558 data_s:0.532 smp/s:16 mem_gb:29.76
+Training:  16%|█▌        | 1600/10000 [1:49:08<9:41:13,  4.15s/step]INFO 2026-07-23 18:24:41 ot_train.py:658 step:52K smpl:3M ep:75K epch:15.12 loss:0.021 grdn:0.244 lr:5.0e-06 updt_s:3.562 data_s:0.529 smp/s:16 mem_gb:29.76
+Training:  18%|█▊        | 1800/10000 [2:02:45<9:31:41,  4.18s/step]INFO 2026-07-23 18:38:18 ot_train.py:658 step:52K smpl:3M ep:75K epch:15.18 loss:0.021 grdn:0.255 lr:4.7e-06 updt_s:3.563 data_s:0.525 smp/s:16 mem_gb:29.76
+Training:  20%|██        | 2000/10000 [2:16:24<8:35:43,  3.87s/step]INFO 2026-07-23 18:51:57 ot_train.py:658 step:52K smpl:3M ep:75K epch:15.24 loss:0.021 grdn:0.243 lr:4.5e-06 updt_s:3.574 data_s:0.517 smp/s:16 mem_gb:29.74
+Training:  22%|██▏       | 2200/10000 [2:30:04<9:15:56,  4.28s/step]INFO 2026-07-23 19:05:37 ot_train.py:658 step:52K smpl:3M ep:75K epch:15.30 loss:0.021 grdn:0.238 lr:4.3e-06 updt_s:3.597 data_s:0.502 smp/s:16 mem_gb:29.75
+Training:  24%|██▍       | 2400/10000 [2:43:43<8:58:20,  4.25s/step]INFO 2026-07-23 19:19:16 ot_train.py:658 step:52K smpl:3M ep:76K epch:15.36 loss:0.020 grdn:0.233 lr:4.1e-06 updt_s:3.596 data_s:0.501 smp/s:16 mem_gb:29.76
+Training:  26%|██▌       | 2600/10000 [2:57:18<8:11:26,  3.98s/step]INFO 2026-07-23 19:32:51 ot_train.py:658 step:53K smpl:3M ep:76K epch:15.42 loss:0.021 grdn:0.251 lr:3.9e-06 updt_s:3.574 data_s:0.499 smp/s:16 mem_gb:29.76
+Training:  28%|██▊       | 2800/10000 [3:10:59<8:07:37,  4.06s/step]INFO 2026-07-23 19:46:32 ot_train.py:658 step:53K smpl:3M ep:76K epch:15.47 loss:0.020 grdn:0.238 lr:3.7e-06 updt_s:3.595 data_s:0.509 smp/s:16 mem_gb:29.76
+Training:  30%|███       | 3000/10000 [3:24:32<8:06:15,  4.17s/step]INFO 2026-07-23 20:00:05 ot_train.py:658 step:53K smpl:3M ep:77K epch:15.53 loss:0.020 grdn:0.230 lr:3.5e-06 updt_s:3.537 data_s:0.526 smp/s:16 mem_gb:29.76
+Training:  32%|███▏      | 3200/10000 [3:38:06<7:35:09,  4.02s/step]INFO 2026-07-23 20:13:39 ot_train.py:658 step:53K smpl:3M ep:77K epch:15.59 loss:0.020 grdn:0.233 lr:3.3e-06 updt_s:3.541 data_s:0.529 smp/s:16 mem_gb:29.75
+Training:  34%|███▍      | 3400/10000 [3:51:42<7:07:09,  3.88s/step]INFO 2026-07-23 20:27:15 ot_train.py:658 step:53K smpl:3M ep:77K epch:15.65 loss:0.021 grdn:0.253 lr:3.1e-06 updt_s:3.555 data_s:0.526 smp/s:16 mem_gb:29.75
+Training:  36%|███▌      | 3600/10000 [4:05:20<7:27:17,  4.19s/step]INFO 2026-07-23 20:40:53 ot_train.py:658 step:54K smpl:3M ep:77K epch:15.71 loss:0.019 grdn:0.239 lr:2.9e-06 updt_s:3.560 data_s:0.528 smp/s:16 mem_gb:29.75
+Training:  38%|███▊      | 3800/10000 [4:19:00<7:12:57,  4.19s/step]INFO 2026-07-23 20:54:32 ot_train.py:658 step:54K smpl:3M ep:78K epch:15.77 loss:0.020 grdn:0.239 lr:2.7e-06 updt_s:3.568 data_s:0.527 smp/s:16 mem_gb:29.75
+Training:  40%|████      | 4000/10000 [4:32:32<6:29:52,  3.90s/step]INFO 2026-07-23 21:08:05 ot_train.py:658 step:54K smpl:3M ep:78K epch:15.83 loss:0.020 grdn:0.247 lr:2.6e-06 updt_s:3.535 data_s:0.525 smp/s:16 mem_gb:29.75
+Training:  42%|████▏     | 4200/10000 [4:46:11<6:24:24,  3.98s/step]INFO 2026-07-23 21:21:44 ot_train.py:658 step:54K smpl:3M ep:78K epch:15.89 loss:0.021 grdn:0.259 lr:2.4e-06 updt_s:3.567 data_s:0.529 smp/s:16 mem_gb:29.74
+Training:  44%|████▍     | 4400/10000 [4:59:51<6:19:47,  4.07s/step]INFO 2026-07-23 21:35:24 ot_train.py:658 step:54K smpl:3M ep:79K epch:15.94 loss:0.020 grdn:0.250 lr:2.2e-06 updt_s:3.573 data_s:0.527 smp/s:16 mem_gb:29.76
+Training:  46%|████▌     | 4600/10000 [5:13:34<6:10:06,  4.11s/step]INFO 2026-07-23 21:49:06 ot_train.py:658 step:55K smpl:3M ep:79K epch:16.00 loss:0.020 grdn:0.254 lr:2.1e-06 updt_s:3.583 data_s:0.527 smp/s:16 mem_gb:29.75
+Training:  48%|████▊     | 4800/10000 [5:27:10<5:56:38,  4.12s/step]INFO 2026-07-23 22:02:42 ot_train.py:658 step:55K smpl:4M ep:79K epch:16.06 loss:0.020 grdn:0.236 lr:1.9e-06 updt_s:3.550 data_s:0.529 smp/s:16 mem_gb:29.76
+Training:  50%|█████     | 5000/10000 [5:40:52<5:53:54,  4.25s/step]INFO 2026-07-23 22:16:25 ot_train.py:658 step:55K smpl:4M ep:79K epch:16.12 loss:0.020 grdn:0.251 lr:1.8e-06 updt_s:3.586 data_s:0.527 smp/s:16 mem_gb:29.76
+INFO 2026-07-23 22:16:25 ot_train.py:704 Checkpoint policy after step 55000
+INFO 2026-07-23 22:16:44 in_yaml.py:1414 Saved verified rolling checkpoint step=55000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00
+Training:  52%|█████▏    | 5200/10000 [5:54:47<5:38:31,  4.23s/step]INFO 2026-07-23 22:30:20 ot_train.py:658 step:55K smpl:4M ep:80K epch:16.18 loss:0.020 grdn:0.240 lr:1.7e-06 updt_s:3.555 data_s:0.527 smp/s:16 mem_gb:29.76
+Training:  54%|█████▍    | 5400/10000 [6:08:26<5:09:12,  4.03s/step]INFO 2026-07-23 22:43:59 ot_train.py:658 step:55K smpl:4M ep:80K epch:16.24 loss:0.020 grdn:0.234 lr:1.5e-06 updt_s:3.565 data_s:0.526 smp/s:16 mem_gb:29.75
+Training:  56%|█████▌    | 5600/10000 [6:22:06<5:00:51,  4.10s/step]INFO 2026-07-23 22:57:39 ot_train.py:658 step:56K smpl:4M ep:80K epch:16.30 loss:0.019 grdn:0.236 lr:1.4e-06 updt_s:3.572 data_s:0.526 smp/s:16 mem_gb:29.75
+Training:  58%|█████▊    | 5800/10000 [6:35:44<4:48:18,  4.12s/step]INFO 2026-07-23 23:11:17 ot_train.py:658 step:56K smpl:4M ep:81K epch:16.35 loss:0.019 grdn:0.239 lr:1.3e-06 updt_s:3.562 data_s:0.526 smp/s:16 mem_gb:29.75
+Training:  60%|██████    | 6000/10000 [6:49:14<4:19:45,  3.90s/step]INFO 2026-07-23 23:24:47 ot_train.py:658 step:56K smpl:4M ep:81K epch:16.41 loss:0.020 grdn:0.248 lr:1.2e-06 updt_s:3.522 data_s:0.528 smp/s:16 mem_gb:29.76
+Training:  62%|██████▏   | 6200/10000 [7:02:52<4:26:32,  4.21s/step]INFO 2026-07-23 23:38:24 ot_train.py:658 step:56K smpl:4M ep:81K epch:16.47 loss:0.020 grdn:0.248 lr:1.1e-06 updt_s:3.560 data_s:0.528 smp/s:16 mem_gb:29.76
+Training:  64%|██████▍   | 6400/10000 [7:16:28<3:58:51,  3.98s/step]INFO 2026-07-23 23:52:01 ot_train.py:658 step:56K smpl:4M ep:81K epch:16.53 loss:0.019 grdn:0.242 lr:9.5e-07 updt_s:3.557 data_s:0.525 smp/s:16 mem_gb:29.75
+Training:  66%|██████▌   | 6600/10000 [7:30:08<3:51:36,  4.09s/step]INFO 2026-07-24 00:05:40 ot_train.py:658 step:57K smpl:4M ep:82K epch:16.59 loss:0.019 grdn:0.233 lr:8.5e-07 updt_s:3.565 data_s:0.531 smp/s:16 mem_gb:29.74
+Training:  68%|██████▊   | 6800/10000 [7:43:44<3:34:35,  4.02s/step]INFO 2026-07-24 00:19:16 ot_train.py:658 step:57K smpl:4M ep:82K epch:16.65 loss:0.019 grdn:0.245 lr:7.6e-07 updt_s:3.550 data_s:0.529 smp/s:16 mem_gb:29.76
+Training:  70%|███████   | 7000/10000 [7:57:22<3:15:37,  3.91s/step]INFO 2026-07-24 00:32:54 ot_train.py:658 step:57K smpl:4M ep:82K epch:16.71 loss:0.019 grdn:0.241 lr:6.7e-07 updt_s:3.558 data_s:0.530 smp/s:16 mem_gb:29.75
+Training:  72%|███████▏  | 7200/10000 [8:10:55<3:14:37,  4.17s/step]INFO 2026-07-24 00:46:28 ot_train.py:658 step:57K smpl:4M ep:83K epch:16.76 loss:0.019 grdn:0.248 lr:5.9e-07 updt_s:3.541 data_s:0.527 smp/s:16 mem_gb:29.76
+Training:  74%|███████▍  | 7400/10000 [8:24:30<2:51:31,  3.96s/step]INFO 2026-07-24 01:00:03 ot_train.py:658 step:57K smpl:4M ep:83K epch:16.82 loss:0.019 grdn:0.238 lr:5.1e-07 updt_s:3.535 data_s:0.538 smp/s:16 mem_gb:29.76
+Training:  76%|███████▌  | 7600/10000 [8:38:23<2:38:29,  3.96s/step]INFO 2026-07-24 01:13:56 ot_train.py:658 step:58K smpl:4M ep:83K epch:16.88 loss:0.019 grdn:0.242 lr:4.4e-07 updt_s:3.575 data_s:0.587 smp/s:15 mem_gb:29.76
+Training:  78%|███████▊  | 7800/10000 [8:52:00<2:19:37,  3.81s/step]INFO 2026-07-24 01:27:33 ot_train.py:658 step:58K smpl:4M ep:84K epch:16.94 loss:0.020 grdn:0.239 lr:3.7e-07 updt_s:3.539 data_s:0.546 smp/s:16 mem_gb:29.75
+Training:  80%|████████  | 8000/10000 [9:05:46<2:14:03,  4.02s/step]INFO 2026-07-24 01:41:19 ot_train.py:658 step:58K smpl:4M ep:84K epch:17.00 loss:0.019 grdn:0.234 lr:3.1e-07 updt_s:3.560 data_s:0.568 smp/s:16 mem_gb:29.76
+Training:  82%|████████▏ | 8200/10000 [9:19:45<2:06:51,  4.23s/step]INFO 2026-07-24 01:55:18 ot_train.py:658 step:58K smpl:4M ep:84K epch:17.06 loss:0.019 grdn:0.241 lr:2.5e-07 updt_s:3.580 data_s:0.614 smp/s:15 mem_gb:29.76
+Training:  84%|████████▍ | 8400/10000 [9:35:39<2:19:08,  5.22s/step]INFO 2026-07-24 02:11:12 ot_train.py:658 step:58K smpl:4M ep:84K epch:17.12 loss:0.019 grdn:0.237 lr:2.0e-07 updt_s:3.903 data_s:0.863 smp/s:13 mem_gb:29.75
+Training:  86%|████████▌ | 8600/10000 [9:50:48<1:38:28,  4.22s/step]INFO 2026-07-24 02:26:21 ot_train.py:658 step:59K smpl:4M ep:85K epch:17.17 loss:0.019 grdn:0.235 lr:1.6e-07 updt_s:3.588 data_s:0.954 smp/s:14 mem_gb:29.75
+Training:  88%|████████▊ | 8800/10000 [10:04:54<1:23:52,  4.19s/step]INFO 2026-07-24 02:40:26 ot_train.py:658 step:59K smpl:4M ep:85K epch:17.23 loss:0.019 grdn:0.224 lr:1.2e-07 updt_s:3.505 data_s:0.720 smp/s:15 mem_gb:29.74
+Training:  90%|█████████ | 9000/10000 [10:18:57<1:13:48,  4.43s/step]INFO 2026-07-24 02:54:30 ot_train.py:658 step:59K smpl:4M ep:85K epch:17.29 loss:0.019 grdn:0.237 lr:8.4e-08 updt_s:3.549 data_s:0.666 smp/s:15 mem_gb:29.75
+Training:  92%|█████████▏| 9200/10000 [10:33:15<55:27,  4.16s/step]INFO 2026-07-24 03:08:47 ot_train.py:658 step:59K smpl:4M ep:86K epch:17.35 loss:0.019 grdn:0.236 lr:5.7e-08 updt_s:3.606 data_s:0.681 smp/s:15 mem_gb:29.76
+Training:  94%|█████████▍| 9400/10000 [10:49:12<58:42,  5.87s/step]  INFO 2026-07-24 03:24:45 ot_train.py:658 step:59K smpl:4M ep:86K epch:17.41 loss:0.020 grdn:0.230 lr:3.4e-08 updt_s:3.752 data_s:1.033 smp/s:13 mem_gb:29.76
+Training:  96%|█████████▌| 9600/10000 [11:04:17<26:56,  4.04s/step]INFO 2026-07-24 03:39:50 ot_train.py:658 step:60K smpl:4M ep:86K epch:17.47 loss:0.020 grdn:0.237 lr:1.8e-08 updt_s:3.646 data_s:0.877 smp/s:14 mem_gb:29.76
+Training:  98%|█████████▊| 9800/10000 [11:18:16<14:02,  4.21s/step]INFO 2026-07-24 03:53:49 ot_train.py:658 step:60K smpl:4M ep:86K epch:17.53 loss:0.019 grdn:0.239 lr:6.5e-09 updt_s:3.565 data_s:0.625 smp/s:15 mem_gb:29.75
+Training: 100%|██████████| 10000/10000 [11:32:49<00:00,  4.25s/step]INFO 2026-07-24 04:08:22 ot_train.py:658 step:60K smpl:4M ep:87K epch:17.59 loss:0.019 grdn:0.232 lr:9.2e-10 updt_s:3.645 data_s:0.719 smp/s:15 mem_gb:29.75
+INFO 2026-07-24 04:08:22 ot_train.py:704 Checkpoint policy after step 60000
+INFO 2026-07-24 04:08:45 in_yaml.py:1414 Saved verified rolling checkpoint step=60000 slot=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01
+Training: 100%|██████████| 10000/10000 [11:33:12<00:00,  4.16s/step]
+INFO 2026-07-24 04:08:45 ot_train.py:790 End of training

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/requirements.txt

@@ -0,0 +1,118 @@
+lerobot==0.6.1
+tzdata==2026.3
+nvidia-cufile-cu12==1.13.1.3
+wandb==0.27.2
+six==1.17.0
+peft==0.19.1
+nvidia-curand-cu12==10.3.9.90
+nvidia-cuda-runtime-cu12==12.8.90
+datasets==4.8.5
+attrs==26.1.0
+cloudpickle==3.1.2
+packaging==25.0
+nvidia-cublas-cu12==12.8.4.1
+cmake==4.1.3
+psutil==7.2.2
+pydantic==2.13.4
+GitPython==3.1.51
+pillow==12.2.0
+pytz==2026.2
+cuda-pathfinder==1.2.2
+nvidia-cusparse-cu12==12.5.8.93
+typing_extensions==4.15.0
+urllib3==2.7.0
+pydantic_core==2.46.4
+draccus==0.10.0
+mypy_extensions==1.1.0
+dill==0.4.1
+typer==0.26.8
+decord==0.6.0
+tqdm==4.68.4
+triton==3.6.0
+idna==3.18
+requests==2.34.2
+nvidia-cudnn-cu12==9.19.0.56
+mdurl==0.1.2
+shellingham==1.5.4
+zipp==4.1.0
+timm==1.0.28
+torch==2.11.0+cu128
+tokenizers==0.22.2
+nvidia-cusparselt-cu12==0.7.1
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cusolver-cu12==11.7.3.90
+opencv-python-headless==4.13.0.92
+httpx==0.28.1
+Jinja2==3.1.6
+einops==0.8.2
+mergedeep==1.3.4
+platformdirs==4.10.0
+nvidia-nvjitlink-cu12==12.8.93
+lerobot==0.6.0
+PyYAML==6.0.3
+accelerate==1.14.0
+anyio==4.14.2
+gymnasium==1.3.0
+pip==26.1.2
+huggingface_hub==1.23.0
+diffusers==0.35.2
+annotated-doc==0.0.4
+pyarrow==25.0.0
+python-dateutil==2.9.0.post0
+termcolor==3.3.0
+nvidia-nccl-cu12==2.28.9
+sentry-sdk==2.65.0
+torchcodec==0.11.1
+pyyaml-include==1.4.1
+multiprocess==0.70.19
+wrapt==2.2.2
+typing-inspection==0.4.2
+nvidia-cuda-nvrtc-cu12==12.8.93
+Pygments==2.20.0
+nvidia-cuda-cupti-cu12==12.8.90
+certifi==2026.6.17
+setuptools==78.1.0
+charset-normalizer==3.4.9
+propcache==0.5.2
+multidict==6.7.1
+regex==2026.7.10
+frozenlist==1.8.0
+smmap==5.0.3
+hf-xet==1.5.1
+torchvision==0.26.0+cu128
+gitdb==4.0.12
+transformers==5.5.4
+rich==15.0.0
+cuda-bindings==12.9.4
+protobuf==7.35.1
+aiosignal==1.4.0
+jsonlines==4.0.0
+yarl==1.24.2
+filelock==3.29.0
+av==15.1.0
+networkx==3.6.1
+sympy==1.14.0
+wheel==0.47.0
+nvidia-nvtx-cu12==12.8.90
+xxhash==3.8.1
+toml==0.10.2
+MarkupSafe==3.0.3
+pandas==2.3.3
+safetensors==0.8.0
+click==8.4.2
+fsspec==2026.2.0
+dm-tree==0.1.10
+absl-py==2.5.0
+Farama-Notifications==0.0.6
+mpmath==1.3.0
+importlib_metadata==9.0.0
+markdown-it-py==4.2.0
+h11==0.16.0
+typing-inspect==0.9.0
+httpcore==1.0.9
+annotated-types==0.7.0
+numpy==2.2.6
+nvidia-nvshmem-cu12==3.4.5
+aiohappyeyeballs==2.7.1
+cuda-toolkit==12.8.1
+aiohttp==3.14.1

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/wandb-metadata.json

@@ -0,0 +1,46 @@
+{
+  "os": "Linux-6.8.0-55-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-23T16:35:01.611488Z",
+  "args": [
+    "--config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/train_config.json",
+    "--resume=true"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePath": "clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePathLocal": "clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline",
+  "host": "ext_minje-box",
+  "executable": "/home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 2,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "1327086579712"
+    }
+  },
+  "memory": {
+    "total": "2164176568320"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "q52i38bd1zs4dxfa7y10j3r5h3dxa68o"
+}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_timestamp":1.7848661021682887e+09,"train/steps":60000,"train/samples":3840000,"train/grad_norm":0.23239044323563576,"train/lr":9.22312585679652e-10,"_runtime":83910,"train/update_s":3.6448144912550924,"train/episodes":86694.41811262691,"train/loss":0.019211085438728334,"train/gpu_mem_gb":29.75469082355499,"_step":60000,"train/samples_per_s":14.665967630170226,"_wandb":{"runtime":83910},"train/dataloading_s":0.7190298600948881,"train/epochs":17.58507466787564}

Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/logs/debug-core.log

@@ -0,0 +1,71 @@
+{"time":"2026-07-23T16:35:01.682970498Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpa9piztjm/port-1456.txt","pid":1456,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.682978333Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmplz0alzhu/port-1457.txt","pid":1457,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.682985044Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpwunx93cw/port-1458.txt","pid":1458,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.6829908Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpf4abll73/port-1459.txt","pid":1459,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.68360802Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1456}
+{"time":"2026-07-23T16:35:01.683616029Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1457}
+{"time":"2026-07-23T16:35:01.683634398Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1458}
+{"time":"2026-07-23T16:35:01.683603455Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1456-2506-1311874521/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.68361302Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1457-2507-772500566/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.683630442Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1458-2509-4014368169/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.683627313Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1459-2508-1604368573/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:01.683696179Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1459}
+{"time":"2026-07-23T16:35:01.832927644Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.833171958Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.833208155Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.833275727Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.837817931Z","level":"INFO","msg":"handleInformInit: received","streamId":"rs1egwv8","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.837959217Z","level":"INFO","msg":"handleInformInit: received","streamId":"2vl3enlf","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.838627327Z","level":"INFO","msg":"handleInformInit: received","streamId":"jeqgwsec","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.838419673Z","level":"INFO","msg":"handleInformInit: received","streamId":"x6da8gup","id":"1(@)"}
+{"time":"2026-07-23T16:35:01.918297888Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpuyiti7pv/port-1460.txt","pid":1460,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-07-23T16:35:01.91863949Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1460}
+{"time":"2026-07-23T16:35:01.918628695Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1460-2829-1778593576/socket","Net":"unix"}}
+{"time":"2026-07-23T16:35:02.074586625Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.079634061Z","level":"INFO","msg":"handleInformInit: received","streamId":"brma7856","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.303790584Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2vl3enlf","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.30665649Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"x6da8gup","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.307741694Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"jeqgwsec","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.32303733Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rs1egwv8","id":"1(@)"}
+{"time":"2026-07-23T16:35:02.493177984Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"brma7856","id":"1(@)"}
+{"time":"2026-07-23T16:35:08.314840933Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"pomu74zdcjwn"}
+{"time":"2026-07-23T16:35:08.415606838Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"u8kbql7lf4bh"}
+{"time":"2026-07-23T16:35:08.420996327Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"bhwn826heh2c"}
+{"time":"2026-07-23T16:35:08.623677864Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"0dnh3nypnhfm"}
+{"time":"2026-07-23T16:35:09.044152431Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4z6b40zq3yfu"}
+{"time":"2026-07-23T18:11:27.126647161Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.126779892Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T18:11:27.12683822Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1457-2507-772500566/socket","Net":"unix"}}
+{"time":"2026-07-23T18:11:27.126852003Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.126886112Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.126932287Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.378619144Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.37874002Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.378802617Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-23T18:11:27.378812716Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-23T18:11:27.378878838Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1456-2506-1311874521/socket","Net":"unix"}}
+{"time":"2026-07-23T18:11:27.378895674Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-23T18:11:35.494138091Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T18:11:35.494189389Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T18:11:35.494201536Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-23T18:12:28.994694228Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-23T18:12:28.994728432Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-23T18:12:28.994736061Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-24T04:08:45.417129571Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.417350121Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-24T04:08:45.417341706Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.417373499Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.41745467Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-24T04:08:45.417448026Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1458-2509-4014368169/socket","Net":"unix"}}
+{"time":"2026-07-24T04:08:47.515259401Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-24T04:08:47.515290273Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-24T04:08:47.515301151Z","level":"INFO","msg":"server is closed"}
+{"time":"2026-07-24T04:10:06.98288621Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-07-24T04:10:06.983062676Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-07-24T04:10:06.983136342Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-07-24T04:10:06.983144044Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-07-24T04:10:06.98322588Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1459-2508-1604368573/socket","Net":"unix"}}
+{"time":"2026-07-24T04:10:06.983271294Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-07-24T04:10:23.060371611Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-07-24T04:10:23.060399527Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-07-24T04:10:23.060415126Z","level":"INFO","msg":"server is closed"}