Add files using upload-large-folder tool

.gitattributes

@@ -65,3 +65,9 @@ Abs_6D[[:space:]]+[[:space:]]DeltaEE/restarts/20260719_231636_rerun_save30k60k/L
 Abs_6D[[:space:]]+[[:space:]]DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/wandb/run-20260719_231652-r3cz38bm/run-r3cz38bm.wandb filter=lfs diff=lfs merge=lfs -text
 Abs_6D[[:space:]]+[[:space:]]DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260719_231652-bepgb8uu/run-bepgb8uu.wandb filter=lfs diff=lfs merge=lfs -text
 Abs_6D[[:space:]]+[[:space:]]DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K.interrupted_reboot_20260719T051924/wandb/run-20260717_154326-sz25gl12/run-sz25gl12.wandb filter=lfs diff=lfs merge=lfs -text
+Abs_6D[[:space:]]+[[:space:]]DeltaEE/Baseline/wandb/run-20260719_231648-h160hqm2/run-h160hqm2.wandb filter=lfs diff=lfs merge=lfs -text
+Abs_6D[[:space:]]+[[:space:]]DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_10K/wandb/run-20260719_231652-ny0tx2de/run-ny0tx2de.wandb filter=lfs diff=lfs merge=lfs -text
+Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260723_005850-21sn30p5/run-21sn30p5.wandb filter=lfs diff=lfs merge=lfs -text
+Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_005850-rd8tz18f/run-rd8tz18f.wandb filter=lfs diff=lfs merge=lfs -text
+Abs_6D/DiT_Layer/8/Baseline/wandb/run-20260722_020800-21sn30p5/run-21sn30p5.wandb filter=lfs diff=lfs merge=lfs -text
+Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260722_020800-rd8tz18f/run-rd8tz18f.wandb filter=lfs diff=lfs merge=lfs -text

Abs_6D + DeltaEE/Baseline/wandb/run-20260719_231648-h160hqm2/run-h160hqm2.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d926bf5ffe9169642d41ebb3007884a9928a8da16f2de033f7e5aa3ce65cd1e
+size 62128128

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_10K/action_space_manifest.json

@@ -0,0 +1,205 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d+ee_delta",
+  "action_views": [
+    "ee_abs_rot6d",
+    "ee_delta"
+  ],
+  "source_feature": null,
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    16
+  ],
+  "names": [
+    "ee_abs_rot6d.x",
+    "ee_abs_rot6d.y",
+    "ee_abs_rot6d.z",
+    "ee_abs_rot6d.r_col0_x",
+    "ee_abs_rot6d.r_col0_y",
+    "ee_abs_rot6d.r_col0_z",
+    "ee_abs_rot6d.r_col1_x",
+    "ee_abs_rot6d.r_col1_y",
+    "ee_abs_rot6d.r_col1_z",
+    "ee_abs_rot6d.gripper",
+    "ee_delta.delta_x",
+    "ee_delta.delta_y",
+    "ee_delta.delta_z",
+    "ee_delta.delta_axis_angle_x",
+    "ee_delta.delta_axis_angle_y",
+    "ee_delta.delta_axis_angle_z"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "75b95092e5fd9a1b16aa6dc7a478883927d7c7cce76c26b56b290b675ae2311a",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0,
+      -0.09827068448066711,
+      -0.13628354668617249,
+      -0.12191051989793777,
+      -2.431101083755493,
+      -0.3816019296646118,
+      -0.5094270706176758
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033,
+      0.11647262424230576,
+      0.13701310753822327,
+      0.13330328464508057,
+      1.7823294401168823,
+      0.5954466462135315,
+      0.6865788102149963
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632,
+      0.0018730097199051647,
+      -0.0015708771114029225,
+      -0.001553903484685181,
+      0.014420973441752116,
+      0.00961362082315661,
+      -0.0017634798849202702
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817,
+      0.015762732601462,
+      0.010587306860662312,
+      0.01180823404609393,
+      0.10086023299718701,
+      0.04747329757360547,
+      0.14801097710214692
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10,
+      -0.02190519006673929,
+      -0.02092386152118027,
+      -0.026422736164554015,
+      -0.11497312558112925,
+      -0.025678747253255536,
+      -0.17124583807766966
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10,
+      -0.013015102506385306,
+      -0.011160732577261764,
+      -0.012954131824952583,
+      -0.056165101267083004,
+      -0.01548487039073523,
+      -0.15972996251089885
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346,
+      0.0005058764599893245,
+      4.561956601090609e-06,
+      4.945756498042436e-05,
+      0.0001528964582560528,
+      0.00017815767540355586,
+      -0.0002718872707462707
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394,
+      0.016358020650084733,
+      0.003987531854121596,
+      0.006191431456332113,
+      0.11275584021509974,
+      0.06100667096893492,
+      0.15292656959583054
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277,
+      0.03626038003704021,
+      0.008814053033445103,
+      0.012299866693624668,
+      0.17731576759283033,
+      0.12117986440120965,
+      0.16474415715691368
+    ]
+  }
+}

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_10K/resume_dgx_20260722_gpu0_independent_nf69vc3o.log

@@ -0,0 +1,200 @@
+INFO 2026-07-22 03:19:55 ot_train.py:272 {'batch_size': 128,
+ 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + '
+                    'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000',
+ 'cudnn_deterministic': False,
+ 'dataset': {'depth_output_unit': 'mm',
+             'episodes': None,
+             'eval_split': 0.0,
+             'image_transforms': {'enable': False,
+                                  'max_num_transforms': 3,
+                                  'random_order': False,
+                                  'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
+                                                                            5.0],
+                                                                'translate': [0.05,
+                                                                              0.05]},
+                                                     'type': 'RandomAffine',
+                                                     'weight': 1.0},
+                                          'brightness': {'kwargs': {'brightness': [0.8,
+                                                                                   1.2]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'contrast': {'kwargs': {'contrast': [0.8,
+                                                                               1.2]},
+                                                       'type': 'ColorJitter',
+                                                       'weight': 1.0},
+                                          'hue': {'kwargs': {'hue': [-0.05,
+                                                                     0.05]},
+                                                  'type': 'ColorJitter',
+                                                  'weight': 1.0},
+                                          'saturation': {'kwargs': {'saturation': [0.5,
+                                                                                   1.5]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'sharpness': {'kwargs': {'sharpness': [0.5,
+                                                                                 1.5]},
+                                                        'type': 'SharpnessJitter',
+                                                        'weight': 1.0}}},
+             'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2',
+             'return_uint8': False,
+             'revision': None,
+             'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2',
+             'streaming': False,
+             'use_imagenet_stats': True,
+             'video_backend': 'torchcodec'},
+ 'env': None,
+ 'env_eval_freq': 20000,
+ 'eval': {'batch_size': 50,
+          'n_episodes': 50,
+          'recording': False,
+          'recording_private': False,
+          'recording_repo_id': None,
+          'use_async_envs': True},
+ 'eval_steps': 0,
+ 'job': {'detach': False,
+         'image': 'huggingface/lerobot-gpu:latest',
+         'tags': [],
+         'target': None,
+         'timeout': '2d'},
+ 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K',
+ 'log_freq': 200,
+ 'max_eval_samples': 0,
+ 'num_workers': 4,
+ 'optimizer': {'betas': [0.9, 0.999],
+               'eps': 1e-08,
+               'grad_clip_norm': 1.0,
+               'lr': 0.0001,
+               'type': 'adamw',
+               'weight_decay': 1e-05},
+ 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + '
+               'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K',
+ 'peft': None,
+ 'persistent_workers': True,
+ 'policy': {'action_decode_transform': None,
+            'action_expert_num_layers': None,
+            'balance_dataset_weights': True,
+            'balance_trajectory_weights': True,
+            'base_model_path': 'nvidia/GR00T-N1.7-3B',
+            'batch_size': 32,
+            'chunk_size': 16,
+            'dataloader_num_workers': 8,
+            'dataset_paths': None,
+            'device': 'cuda',
+            'embodiment_tag': 'new_embodiment',
+            'image_size': [256, 256],
+            'input_features': {'observation.images.guide': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.right_shoulder': {'shape': [3,
+                                                                               224,
+                                                                               224],
+                                                                     'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.wrist': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.state': {'shape': [16],
+                                                     'type': <FeatureType.STATE: 'STATE'>}},
+            'license': None,
+            'lora_alpha': 16,
+            'lora_dropout': 0.1,
+            'lora_full_model': False,
+            'lora_rank': 0,
+            'max_action_dim': 132,
+            'max_state_dim': 132,
+            'max_steps': 10000,
+            'model_params_fp32': True,
+            'n_action_steps': 16,
+            'n_obs_steps': 1,
+            'normalization_mapping': {'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
+            'num_inference_timesteps': None,
+            'optimizer_betas': [0.9, 0.999],
+            'optimizer_eps': 1e-08,
+            'optimizer_lr': 0.0001,
+            'optimizer_weight_decay': 1e-05,
+            'output_dir': './tmp/gr00t',
+            'output_features': {'action': {'shape': [16],
+                                           'type': <FeatureType.ACTION: 'ACTION'>}},
+            'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D '
+                               '+ '
+                               'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model',
+            'pretrained_revision': None,
+            'private': None,
+            'push_to_hub': False,
+            'relative_exclude_joints': [],
+            'repo_id': None,
+            'report_to': 'wandb',
+            'resume': False,
+            'rkd_angle_weight': 2.0,
+            'rkd_distance_weight': 1.0,
+            'rkd_enabled': True,
+            'rkd_fm_loss_weight': 1.0,
+            'rkd_loss_weight': 1.0,
+            'rkd_student_attention_hidden_dim': 512,
+            'rkd_student_attention_output_dim': 512,
+            'rkd_student_pooling': 'attention',
+            'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84',
+            'rtc_ramp_rate': None,
+            'save_steps': 1000,
+            'tags': None,
+            'tokenizer_assets_repo': None,
+            'tune_diffusion_model': True,
+            'tune_llm': False,
+            'tune_projector': True,
+            'tune_top_llm_layers': 0,
+            'tune_visual': False,
+            'tune_vlln': True,
+            'type': 'groot_rkd',
+            'use_amp': False,
+            'use_bf16': True,
+            'use_flash_attention': False,
+            'use_peft': False,
+            'use_relative_actions': False,
+            'video_backend': 'decord',
+            'warmup_ratio': 0.05,
+            'warmup_steps': 6000},
+ 'prefetch_factor': 4,
+ 'rename_map': {},
+ 'resume': True,
+ 'reward_model': None,
+ 'sample_weighting': None,
+ 'save_checkpoint': True,
+ 'save_checkpoint_to_hub': False,
+ 'save_freq': 20000,
+ 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'},
+ 'seed': 42,
+ 'steps': 60000,
+ 'tolerance_s': 0.0001,
+ 'use_policy_training_preset': True,
+ 'wandb': {'add_tags': True,
+           'disable_artifact': True,
+           'enable': True,
+           'entity': None,
+           'mode': None,
+           'notes': None,
+           'project': 'lerobot',
+           'run_id': 'nf69vc3o'}}
+INFO 2026-07-22 03:19:57 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-22 03:19:57 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/nf69vc3o
+INFO 2026-07-22 03:19:57 ot_train.py:298 Creating dataset
+INFO 2026-07-22 03:20:00 ot_train.py:332 Creating policy
+INFO 2026-07-22 03:20:00 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model
+INFO 2026-07-22 03:20:00 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+Loading weights from local directory
+INFO 2026-07-22 03:20:13 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-22 03:20:16 ot_train.py:103 INSIGHT phase transition: index=1 name=fm_only range=[10000,60000) trainable_params=1620515968
+INFO 2026-07-22 03:20:16 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K
+INFO 2026-07-22 03:20:16 ot_train.py:446 cfg.steps=60000 (60K)
+INFO 2026-07-22 03:20:16 ot_train.py:447 dataset.num_frames=218367 (218K)
+INFO 2026-07-22 03:20:16 ot_train.py:448 dataset.num_episodes=4930
+INFO 2026-07-22 03:20:16 ot_train.py:451 Effective batch size: 128 x 1 = 128
+INFO 2026-07-22 03:20:16 ot_train.py:452 num_learnable_params=1620515968 (2B)
+INFO 2026-07-22 03:20:16 ot_train.py:453 num_total_params=3145712033 (3B)
+INFO 2026-07-22 03:20:16 ot_train.py:494 Resuming data order at epoch 26, sample 82688
+
+

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_10K/source_config.yaml

@@ -0,0 +1,65 @@
+# INSIGHT GR00T N1.7 RKD training with LAP-style early RKD guidance.
+# Common RKD values live in the top-level policy; phases contain overrides only.
+insight:
+  # Composite policy action: ee_abs_rot6d 10D + ee_delta 6D (duplicate gripper removed)
+  action_space: ee_abs_rot6d+ee_delta
+  # Options: coarse | detailed
+  prompt_set: detailed
+  save_steps: ["60K"]
+  phases:
+    - name: lap_rkd_linear_decay_10K
+      until_step: 10K
+      policy:
+        # Inherits RKD=1.0 below and linearly decays it to zero.
+        rkd_loss_weight_end: 0.0
+
+    - name: fm_only
+      until_step: 60K
+      policy:
+        rkd_enabled: false
+        rkd_loss_weight: 0.0
+
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
+
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  # LAP Stage-2 warmup scaled to this 60K run: exactly 6K updates.
+  warmup_steps: 6000
+
+  # VLM
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  # Action Expert
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+
+  # Frozen RD TimewarpVAE teacher: ee_abs_rot6d 10D + ee_delta 6D (gripper deduplicated).
+  rkd_enabled: true
+  rkd_vae_checkpoint: ${HOME}/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84
+  rkd_student_pooling: attention  # attention | mean
+  rkd_student_attention_hidden_dim: 512
+  rkd_student_attention_output_dim: 512
+  rkd_fm_loss_weight: 1.0
+  rkd_loss_weight: 1.0
+  rkd_distance_weight: 1.0
+  rkd_angle_weight: 2.0
+
+seed: 42
+# Single GPU/process, so this is also the global batch size.
+batch_size: 128
+steps: 60000
+log_freq: 200
+output_dir: "${HOME}/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_10K"
+job_name: INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K
+
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_10K/wandb/run-20260719_231652-ny0tx2de/run-ny0tx2de.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d012ddb84585b76ead6f2718220c6fc1fb69cbe1e2073e420a5c27d5a978b7cc
+size 62390272

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/action_space_manifest.json

@@ -0,0 +1,205 @@
+{
+  "schema_version": 2,
+  "action_space": "ee_abs_rot6d+ee_delta",
+  "action_views": [
+    "ee_abs_rot6d",
+    "ee_delta"
+  ],
+  "source_feature": null,
+  "canonical_feature": "action",
+  "dtype": "float32",
+  "shape": [
+    16
+  ],
+  "names": [
+    "ee_abs_rot6d.x",
+    "ee_abs_rot6d.y",
+    "ee_abs_rot6d.z",
+    "ee_abs_rot6d.r_col0_x",
+    "ee_abs_rot6d.r_col0_y",
+    "ee_abs_rot6d.r_col0_z",
+    "ee_abs_rot6d.r_col1_x",
+    "ee_abs_rot6d.r_col1_y",
+    "ee_abs_rot6d.r_col1_z",
+    "ee_abs_rot6d.gripper",
+    "ee_delta.delta_x",
+    "ee_delta.delta_y",
+    "ee_delta.delta_z",
+    "ee_delta.delta_axis_angle_x",
+    "ee_delta.delta_axis_angle_y",
+    "ee_delta.delta_axis_angle_z"
+  ],
+  "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
+  "dataset_revision": "v3.0",
+  "dataset_total_frames": 218367,
+  "dataset_total_episodes": 4930,
+  "dataset_fps": 10,
+  "stats_sha256": "75b95092e5fd9a1b16aa6dc7a478883927d7c7cce76c26b56b290b675ae2311a",
+  "stats": {
+    "min": [
+      -0.21784618496894836,
+      -0.4474811851978302,
+      0.1702737659215927,
+      -0.9999986290931702,
+      -0.9999991655349731,
+      -0.9999999403953552,
+      -0.9999990463256836,
+      -0.9999998211860657,
+      -0.9999988079071045,
+      0.0,
+      -0.09827068448066711,
+      -0.13628354668617249,
+      -0.12191051989793777,
+      -2.431101083755493,
+      -0.3816019296646118,
+      -0.5094270706176758
+    ],
+    "max": [
+      0.6283698678016663,
+      0.4420371949672699,
+      1.051027774810791,
+      0.9999999403953552,
+      0.9999995231628418,
+      1.0,
+      0.9999995231628418,
+      1.0,
+      0.9999996423721313,
+      0.03999999910593033,
+      0.11647262424230576,
+      0.13701310753822327,
+      0.13330328464508057,
+      1.7823294401168823,
+      0.5954466462135315,
+      0.6865788102149963
+    ],
+    "mean": [
+      0.4040228037735655,
+      -0.05247618696989652,
+      0.6993459771218912,
+      0.07011111991957117,
+      -0.03520558904460197,
+      0.14470242846953205,
+      0.05491166606739672,
+      -0.19073058893378805,
+      -0.06988117661871228,
+      0.021167300266524632,
+      0.0018730097199051647,
+      -0.0015708771114029225,
+      -0.001553903484685181,
+      0.014420973441752116,
+      0.00961362082315661,
+      -0.0017634798849202702
+    ],
+    "std": [
+      0.11861927913357073,
+      0.11318943557281885,
+      0.17849160646987516,
+      0.4812528791886511,
+      0.7487196258736938,
+      0.4251126804030219,
+      0.47797229807769137,
+      0.6267695346407992,
+      0.5782954306815554,
+      0.01996590431050817,
+      0.015762732601462,
+      0.010587306860662312,
+      0.01180823404609393,
+      0.10086023299718701,
+      0.04747329757360547,
+      0.14801097710214692
+    ],
+    "count": [
+      218367
+    ],
+    "q01": [
+      0.23988881827972064,
+      -0.10387461883034811,
+      0.6297243923476871,
+      -0.43622841955353464,
+      -0.6885486527617446,
+      -0.14103555388039019,
+      -0.4327210063987595,
+      -0.9304790915716145,
+      -0.3465462258490417,
+      -1.000000013351432e-10,
+      -0.02190519006673929,
+      -0.02092386152118027,
+      -0.026422736164554015,
+      -0.11497312558112925,
+      -0.025678747253255536,
+      -0.17124583807766966
+    ],
+    "q10": [
+      0.3042306436755216,
+      -0.09609991620622624,
+      0.6394063072781145,
+      -0.3351306352998669,
+      -0.5751963015002924,
+      -0.08490621283307866,
+      -0.33849701676224275,
+      -0.8340557712529717,
+      -0.29760290790884514,
+      -1.000000013351432e-10,
+      -0.013015102506385306,
+      -0.011160732577261764,
+      -0.012954131824952583,
+      -0.056165101267083004,
+      -0.01548487039073523,
+      -0.15972996251089885
+    ],
+    "q50": [
+      0.4160225263528291,
+      -0.06267459152989589,
+      0.6811777491060801,
+      0.0543536902230377,
+      -0.14012251058819217,
+      0.04772394280004727,
+      0.03838119138716219,
+      -0.14865169094195635,
+      -0.2073069401172425,
+      0.020772594934870346,
+      0.0005058764599893245,
+      4.561956601090609e-06,
+      4.945756498042436e-05,
+      0.0001528964582560528,
+      0.00017815767540355586,
+      -0.0002718872707462707
+    ],
+    "q90": [
+      0.46641008755679864,
+      0.020233143101890064,
+      0.8095135405259374,
+      0.48172401592474906,
+      0.7398737043070319,
+      0.5970841215636613,
+      0.4674752875999555,
+      0.36281754777672676,
+      0.5033361883834496,
+      0.039998362213352394,
+      0.016358020650084733,
+      0.003987531854121596,
+      0.006191431456332113,
+      0.11275584021509974,
+      0.06100667096893492,
+      0.15292656959583054
+    ],
+    "q99": [
+      0.47320723171849244,
+      0.04114262476135705,
+      0.8467967259270389,
+      0.5867781409980988,
+      0.8529584859257228,
+      0.7953936054341789,
+      0.5768269858732034,
+      0.49353349323389695,
+      0.7168914440776504,
+      0.03999983541667277,
+      0.03626038003704021,
+      0.008814053033445103,
+      0.012299866693624668,
+      0.17731576759283033,
+      0.12117986440120965,
+      0.16474415715691368
+    ]
+  }
+}

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/phase_schedule.json

@@ -0,0 +1,55 @@
+{
+  "schema_version": 1,
+  "step_interval": "[start_step, until_step)",
+  "total_steps": 60000,
+  "optimizer_lifecycle": "single_continuous_optimizer",
+  "scheduler_lifecycle": "single_continuous_scheduler",
+  "phases": [
+    {
+      "index": 0,
+      "name": "lap_rkd_linear_decay_6K",
+      "start_step": 0,
+      "until_step": 6000,
+      "policy": {
+        "rkd_enabled": true,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 1.0,
+        "rkd_loss_weight_end": 0.0,
+        "rkd_loss_weight_schedule": "linear",
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+      },
+      "trainable": {
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_top_llm_layers": 0,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true
+      }
+    },
+    {
+      "index": 1,
+      "name": "fm_only",
+      "start_step": 6000,
+      "until_step": 60000,
+      "policy": {
+        "rkd_enabled": false,
+        "rkd_fm_loss_weight": 1.0,
+        "rkd_loss_weight": 0.0,
+        "rkd_loss_weight_end": null,
+        "rkd_loss_weight_schedule": "linear",
+        "rkd_distance_weight": 1.0,
+        "rkd_angle_weight": 2.0
+      },
+      "trainable": {
+        "tune_llm": false,
+        "tune_visual": false,
+        "tune_top_llm_layers": 0,
+        "tune_projector": true,
+        "tune_diffusion_model": true,
+        "tune_vlln": true
+      }
+    }
+  ]
+}

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/resolved_config.yaml

@@ -0,0 +1,34 @@
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  warmup_steps: 6000
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  rkd_enabled: true
+  rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84
+  rkd_student_pooling: attention
+  rkd_student_attention_hidden_dim: 512
+  rkd_student_attention_output_dim: 512
+  rkd_fm_loss_weight: 1.0
+  rkd_loss_weight: 1.0
+  rkd_distance_weight: 1.0
+  rkd_angle_weight: 2.0
+seed: 42
+batch_size: 128
+steps: 60000
+log_freq: 200
+output_dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K
+job_name: INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/resume_dgx_20260722_fix_gpu0.log

@@ -0,0 +1,210 @@
+INFO 2026-07-21 18:07:58 ot_train.py:272 {'batch_size': 128,
+ 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + '
+                    'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000',
+ 'cudnn_deterministic': False,
+ 'dataset': {'depth_output_unit': 'mm',
+             'episodes': None,
+             'eval_split': 0.0,
+             'image_transforms': {'enable': False,
+                                  'max_num_transforms': 3,
+                                  'random_order': False,
+                                  'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
+                                                                            5.0],
+                                                                'translate': [0.05,
+                                                                              0.05]},
+                                                     'type': 'RandomAffine',
+                                                     'weight': 1.0},
+                                          'brightness': {'kwargs': {'brightness': [0.8,
+                                                                                   1.2]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'contrast': {'kwargs': {'contrast': [0.8,
+                                                                               1.2]},
+                                                       'type': 'ColorJitter',
+                                                       'weight': 1.0},
+                                          'hue': {'kwargs': {'hue': [-0.05,
+                                                                     0.05]},
+                                                  'type': 'ColorJitter',
+                                                  'weight': 1.0},
+                                          'saturation': {'kwargs': {'saturation': [0.5,
+                                                                                   1.5]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'sharpness': {'kwargs': {'sharpness': [0.5,
+                                                                                 1.5]},
+                                                        'type': 'SharpnessJitter',
+                                                        'weight': 1.0}}},
+             'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2',
+             'return_uint8': False,
+             'revision': None,
+             'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2',
+             'streaming': False,
+             'use_imagenet_stats': True,
+             'video_backend': 'torchcodec'},
+ 'env': None,
+ 'env_eval_freq': 20000,
+ 'eval': {'batch_size': 50,
+          'n_episodes': 50,
+          'recording': False,
+          'recording_private': False,
+          'recording_repo_id': None,
+          'use_async_envs': True},
+ 'eval_steps': 0,
+ 'job': {'detach': False,
+         'image': 'huggingface/lerobot-gpu:latest',
+         'tags': [],
+         'target': None,
+         'timeout': '2d'},
+ 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K',
+ 'log_freq': 200,
+ 'max_eval_samples': 0,
+ 'num_workers': 4,
+ 'optimizer': {'betas': [0.9, 0.999],
+               'eps': 1e-08,
+               'grad_clip_norm': 1.0,
+               'lr': 0.0001,
+               'type': 'adamw',
+               'weight_decay': 1e-05},
+ 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + '
+               'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K',
+ 'peft': None,
+ 'persistent_workers': True,
+ 'policy': {'action_decode_transform': None,
+            'balance_dataset_weights': True,
+            'balance_trajectory_weights': True,
+            'base_model_path': 'nvidia/GR00T-N1.7-3B',
+            'batch_size': 32,
+            'chunk_size': 16,
+            'dataloader_num_workers': 8,
+            'dataset_paths': None,
+            'device': 'cuda',
+            'embodiment_tag': 'new_embodiment',
+            'image_size': [256, 256],
+            'input_features': {'observation.images.guide': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.right_shoulder': {'shape': [3,
+                                                                               224,
+                                                                               224],
+                                                                     'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.wrist': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.state': {'shape': [16],
+                                                     'type': <FeatureType.STATE: 'STATE'>}},
+            'license': None,
+            'lora_alpha': 16,
+            'lora_dropout': 0.1,
+            'lora_full_model': False,
+            'lora_rank': 0,
+            'max_action_dim': 132,
+            'max_state_dim': 132,
+            'max_steps': 10000,
+            'model_params_fp32': True,
+            'n_action_steps': 16,
+            'n_obs_steps': 1,
+            'normalization_mapping': {'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
+            'num_inference_timesteps': None,
+            'optimizer_betas': [0.9, 0.999],
+            'optimizer_eps': 1e-08,
+            'optimizer_lr': 0.0001,
+            'optimizer_weight_decay': 1e-05,
+            'output_dir': './tmp/gr00t',
+            'output_features': {'action': {'shape': [16],
+                                           'type': <FeatureType.ACTION: 'ACTION'>}},
+            'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D '
+                               '+ '
+                               'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model',
+            'pretrained_revision': None,
+            'private': None,
+            'push_to_hub': False,
+            'relative_exclude_joints': [],
+            'repo_id': None,
+            'report_to': 'wandb',
+            'resume': False,
+            'rkd_angle_weight': 2.0,
+            'rkd_distance_weight': 1.0,
+            'rkd_enabled': True,
+            'rkd_fm_loss_weight': 1.0,
+            'rkd_loss_weight': 1.0,
+            'rkd_student_attention_hidden_dim': 512,
+            'rkd_student_attention_output_dim': 512,
+            'rkd_student_pooling': 'attention',
+            'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84',
+            'rtc_ramp_rate': None,
+            'save_steps': 1000,
+            'tags': None,
+            'tokenizer_assets_repo': None,
+            'tune_diffusion_model': True,
+            'tune_llm': False,
+            'tune_projector': True,
+            'tune_top_llm_layers': 0,
+            'tune_visual': False,
+            'tune_vlln': True,
+            'type': 'groot_rkd',
+            'use_amp': False,
+            'use_bf16': True,
+            'use_flash_attention': False,
+            'use_peft': False,
+            'use_relative_actions': False,
+            'video_backend': 'decord',
+            'warmup_ratio': 0.05,
+            'warmup_steps': 6000},
+ 'prefetch_factor': 4,
+ 'rename_map': {},
+ 'resume': True,
+ 'reward_model': None,
+ 'sample_weighting': None,
+ 'save_checkpoint': True,
+ 'save_checkpoint_to_hub': False,
+ 'save_freq': 20000,
+ 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'},
+ 'seed': 42,
+ 'steps': 60000,
+ 'tolerance_s': 0.0001,
+ 'use_policy_training_preset': True,
+ 'wandb': {'add_tags': True,
+           'disable_artifact': True,
+           'enable': True,
+           'entity': None,
+           'mode': None,
+           'notes': None,
+           'project': 'lerobot',
+           'run_id': 'r3cz38bm'}}
+INFO 2026-07-21 18:07:59 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-21 18:07:59 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/r3cz38bm
+INFO 2026-07-21 18:07:59 ot_train.py:298 Creating dataset
+INFO 2026-07-21 18:08:01 ot_train.py:332 Creating policy
+INFO 2026-07-21 18:08:01 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model
+INFO 2026-07-21 18:08:01 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+Loading weights from local directory
+Traceback (most recent call last):
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1430, in <module>
+    main()
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1426, in main
+    train_module.main()
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 397, in train
+    logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")
+                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1204, in make_pre_post_processors
+    return original_make_pre_post_processors(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 323, in make_pre_post_processors
+    preprocessor = PolicyProcessorPipeline.from_pretrained(
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 738, in from_pretrained
+    cls._validate_overrides_used(validated_overrides, loaded_config)
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 1254, in _validate_overrides_used
+    raise KeyError(
+KeyError: "Override keys ['normalizer_processor'] do not match any step in the saved configuration. Available step keys: ['rename_observations_processor', 'to_batch_processor', 'rkd_preserve_raw_action', 'groot_n1_7_pack_inputs_v1', 'groot_n1_7_vlm_encode_v1', 'device_processor']. Make sure override keys match exact step class names or registry names."

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/resume_dgx_20260722_gpu0.log

@@ -0,0 +1,277 @@
+INFO 2026-07-21 17:56:01 ot_train.py:272 {'batch_size': 128,
+ 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + '
+                    'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000',
+ 'cudnn_deterministic': False,
+ 'dataset': {'depth_output_unit': 'mm',
+             'episodes': None,
+             'eval_split': 0.0,
+             'image_transforms': {'enable': False,
+                                  'max_num_transforms': 3,
+                                  'random_order': False,
+                                  'tfs': {'affine': {'kwargs': {'degrees': [-5.0,
+                                                                            5.0],
+                                                                'translate': [0.05,
+                                                                              0.05]},
+                                                     'type': 'RandomAffine',
+                                                     'weight': 1.0},
+                                          'brightness': {'kwargs': {'brightness': [0.8,
+                                                                                   1.2]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'contrast': {'kwargs': {'contrast': [0.8,
+                                                                               1.2]},
+                                                       'type': 'ColorJitter',
+                                                       'weight': 1.0},
+                                          'hue': {'kwargs': {'hue': [-0.05,
+                                                                     0.05]},
+                                                  'type': 'ColorJitter',
+                                                  'weight': 1.0},
+                                          'saturation': {'kwargs': {'saturation': [0.5,
+                                                                                   1.5]},
+                                                         'type': 'ColorJitter',
+                                                         'weight': 1.0},
+                                          'sharpness': {'kwargs': {'sharpness': [0.5,
+                                                                                 1.5]},
+                                                        'type': 'SharpnessJitter',
+                                                        'weight': 1.0}}},
+             'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2',
+             'return_uint8': False,
+             'revision': None,
+             'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2',
+             'streaming': False,
+             'use_imagenet_stats': True,
+             'video_backend': 'torchcodec'},
+ 'env': None,
+ 'env_eval_freq': 20000,
+ 'eval': {'batch_size': 50,
+          'n_episodes': 50,
+          'recording': False,
+          'recording_private': False,
+          'recording_repo_id': None,
+          'use_async_envs': True},
+ 'eval_steps': 0,
+ 'job': {'detach': False,
+         'image': 'huggingface/lerobot-gpu:latest',
+         'tags': [],
+         'target': None,
+         'timeout': '2d'},
+ 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K',
+ 'log_freq': 200,
+ 'max_eval_samples': 0,
+ 'num_workers': 4,
+ 'optimizer': {'betas': [0.9, 0.999],
+               'eps': 1e-08,
+               'grad_clip_norm': 1.0,
+               'lr': 0.0001,
+               'type': 'adamw',
+               'weight_decay': 1e-05},
+ 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + '
+               'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K',
+ 'peft': None,
+ 'persistent_workers': True,
+ 'policy': {'action_decode_transform': None,
+            'balance_dataset_weights': True,
+            'balance_trajectory_weights': True,
+            'base_model_path': 'nvidia/GR00T-N1.7-3B',
+            'batch_size': 32,
+            'chunk_size': 16,
+            'dataloader_num_workers': 8,
+            'dataset_paths': None,
+            'device': 'cuda',
+            'embodiment_tag': 'new_embodiment',
+            'image_size': [256, 256],
+            'input_features': {'observation.images.guide': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.right_shoulder': {'shape': [3,
+                                                                               224,
+                                                                               224],
+                                                                     'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.images.wrist': {'shape': [3,
+                                                                      224,
+                                                                      224],
+                                                            'type': <FeatureType.VISUAL: 'VISUAL'>},
+                               'observation.state': {'shape': [16],
+                                                     'type': <FeatureType.STATE: 'STATE'>}},
+            'license': None,
+            'lora_alpha': 16,
+            'lora_dropout': 0.1,
+            'lora_full_model': False,
+            'lora_rank': 0,
+            'max_action_dim': 132,
+            'max_state_dim': 132,
+            'max_steps': 10000,
+            'model_params_fp32': True,
+            'n_action_steps': 16,
+            'n_obs_steps': 1,
+            'normalization_mapping': {'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>,
+                                      'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
+            'num_inference_timesteps': None,
+            'optimizer_betas': [0.9, 0.999],
+            'optimizer_eps': 1e-08,
+            'optimizer_lr': 0.0001,
+            'optimizer_weight_decay': 1e-05,
+            'output_dir': './tmp/gr00t',
+            'output_features': {'action': {'shape': [16],
+                                           'type': <FeatureType.ACTION: 'ACTION'>}},
+            'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D '
+                               '+ '
+                               'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model',
+            'pretrained_revision': None,
+            'private': None,
+            'push_to_hub': False,
+            'relative_exclude_joints': [],
+            'repo_id': None,
+            'report_to': 'wandb',
+            'resume': False,
+            'rkd_angle_weight': 2.0,
+            'rkd_distance_weight': 1.0,
+            'rkd_enabled': True,
+            'rkd_fm_loss_weight': 1.0,
+            'rkd_loss_weight': 1.0,
+            'rkd_student_attention_hidden_dim': 512,
+            'rkd_student_attention_output_dim': 512,
+            'rkd_student_pooling': 'attention',
+            'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84',
+            'rtc_ramp_rate': None,
+            'save_steps': 1000,
+            'tags': None,
+            'tokenizer_assets_repo': None,
+            'tune_diffusion_model': True,
+            'tune_llm': False,
+            'tune_projector': True,
+            'tune_top_llm_layers': 0,
+            'tune_visual': False,
+            'tune_vlln': True,
+            'type': 'groot_rkd',
+            'use_amp': False,
+            'use_bf16': True,
+            'use_flash_attention': False,
+            'use_peft': False,
+            'use_relative_actions': False,
+            'video_backend': 'decord',
+            'warmup_ratio': 0.05,
+            'warmup_steps': 6000},
+ 'prefetch_factor': 4,
+ 'rename_map': {},
+ 'resume': True,
+ 'reward_model': None,
+ 'sample_weighting': None,
+ 'save_checkpoint': True,
+ 'save_checkpoint_to_hub': False,
+ 'save_freq': 20000,
+ 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'},
+ 'seed': 42,
+ 'steps': 60000,
+ 'tolerance_s': 0.0001,
+ 'use_policy_training_preset': True,
+ 'wandb': {'add_tags': True,
+           'disable_artifact': True,
+           'enable': True,
+           'entity': None,
+           'mode': None,
+           'notes': None,
+           'project': 'lerobot',
+           'run_id': 'r3cz38bm'}}
+INFO 2026-07-21 17:56:03 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-21 17:56:03 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/r3cz38bm
+INFO 2026-07-21 17:56:03 ot_train.py:298 Creating dataset
+INFO 2026-07-21 17:56:05 ot_train.py:332 Creating policy
+INFO 2026-07-21 17:56:05 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model
+INFO 2026-07-21 17:56:05 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict...
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 795, in hf_raise_for_status
+    response.raise_for_status()
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/httpx/_models.py", line 829, in raise_for_status
+    raise HTTPStatusError(message, request=request, response=self)
+httpx.HTTPStatusError: Client error '401 Unauthorized' for url 'https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json'
+For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 422, in cached_files
+    hf_hub_download(
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1013, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1165, in _hf_hub_download_to_cache_dir
+    _raise_on_head_call_error(head_call_error, force_download, local_files_only)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1864, in _raise_on_head_call_error
+    raise head_call_error
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1707, in _get_metadata_or_catch_error
+    metadata = get_hf_file_metadata(
+               ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1615, in get_hf_file_metadata
+    response = _httpx_follow_relative_redirects_with_backoff(
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 726, in _httpx_follow_relative_redirects_with_backoff
+    hf_raise_for_status(response)
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 882, in hf_raise_for_status
+    raise _format(RepositoryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e
+huggingface_hub.errors.RepositoryNotFoundError: 401 Client Error. (Request ID: Root=1-6a5fb2b5-6824134c4e397387149d0cce;cac3eee4-7806-45b9-adfe-9e3e32a2e235)
+
+Repository Not Found for url: https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json.
+Please make sure you specified the correct `repo_id` and `repo_type`.
+If you are trying to access a private or gated repo, make sure you are authenticated and your token has the required permissions.
+For more details, see https://huggingface.co/docs/huggingface_hub/authentication
+OAuth token signature verification failed: signature verification failed
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1428, in <module>
+    main()
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1424, in main
+    train_module.main()
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 333, in train
+    }
+      
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1170, in make_policy
+    policy = original_make_policy(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 615, in make_policy
+    policy = policy_cls.from_pretrained(**kwargs)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 226, in from_pretrained
+    return super().from_pretrained(
+           ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/pretrained.py", line 194, in from_pretrained
+    instance = cls(config, **kwargs)
+               ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot_rkd.py", line 44, in __init__
+    super().__init__(config, **kwargs)
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 79, in __init__
+    self._groot_model = self._create_groot_model()
+                        ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 104, in _create_groot_model
+    model = GR00TN17.from_pretrained(
+            ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/groot_n1_7.py", line 915, in from_pretrained
+    pretrained_model = super().from_pretrained(
+                       ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3993, in from_pretrained
+    _adapter_model_path, pretrained_model_name_or_path, adapter_kwargs = maybe_load_adapters(
+                                                                         ^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/integrations/peft.py", line 973, in maybe_load_adapters
+    resolved_config_file = cached_file(
+                           ^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 278, in cached_file
+    file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 454, in cached_files
+    raise OSError(
+OSError: nvidia/GR00T-N1.7-3B is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'
+If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>`

Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/source_config.yaml

@@ -0,0 +1,65 @@
+# INSIGHT GR00T N1.7 RKD training with LAP-style early RKD guidance.
+# Common RKD values live in the top-level policy; phases contain overrides only.
+insight:
+  # Composite policy action: ee_abs_rot6d 10D + ee_delta 6D (duplicate gripper removed)
+  action_space: ee_abs_rot6d+ee_delta
+  # Options: coarse | detailed
+  prompt_set: detailed
+  save_steps: ["60K"]
+  phases:
+    - name: lap_rkd_linear_decay_6K
+      until_step: 6K
+      policy:
+        # Inherits RKD=1.0 below and linearly decays it to zero.
+        rkd_loss_weight_end: 0.0
+
+    - name: fm_only
+      until_step: 60K
+      policy:
+        rkd_enabled: false
+        rkd_loss_weight: 0.0
+
+dataset:
+  repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
+  root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
+
+policy:
+  type: groot_rkd
+  device: cuda
+  chunk_size: 16
+  n_action_steps: 16
+  push_to_hub: false
+  # LAP Stage-2 warmup scaled to this 60K run: exactly 6K updates.
+  warmup_steps: 6000
+
+  # VLM
+  tune_llm: false
+  tune_visual: false
+  tune_top_llm_layers: 0
+  # Action Expert
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+
+  # Frozen RD TimewarpVAE teacher: ee_abs_rot6d 10D + ee_delta 6D (gripper deduplicated).
+  rkd_enabled: true
+  rkd_vae_checkpoint: ${HOME}/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84
+  rkd_student_pooling: attention  # attention | mean
+  rkd_student_attention_hidden_dim: 512
+  rkd_student_attention_output_dim: 512
+  rkd_fm_loss_weight: 1.0
+  rkd_loss_weight: 1.0
+  rkd_distance_weight: 1.0
+  rkd_angle_weight: 2.0
+
+seed: 42
+# Single GPU/process, so this is also the global batch size.
+batch_size: 128
+steps: 60000
+log_freq: 200
+output_dir: "${HOME}/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K"
+job_name: INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K
+
+wandb:
+  enable: true
+  disable_artifact: true

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/LAPstyle_linear_6K.interrupted_reboot_20260719T051924/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_setup.py:_flush():81] Configure stats pid to 499778
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260717_154326-sz25gl12/logs/debug.log
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260717_154326-sz25gl12/logs/debug-internal.log
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:init():766] calling init triggers
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:init():814] starting backend
+2026-07-17 15:43:26,346 INFO    MainThread:499778 [wandb_init.py:init():829] sending inform_init request
+2026-07-17 15:43:26,990 INFO    MainThread:499778 [wandb_init.py:init():834] backend started and connected
+2026-07-17 15:43:26,993 INFO    MainThread:499778 [wandb_init.py:init():904] updated telemetry
+2026-07-17 15:43:27,003 INFO    MainThread:499778 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-17 15:43:27,554 INFO    MainThread:499778 [wandb_init.py:init():972] starting run threads in backend
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-17 15:43:27,754 INFO    MainThread:499778 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/LAPstyle_linear_6K.interrupted_reboot_20260719T051924/wandb/run-20260717_154326-sz25gl12/files/output.log

@@ -0,0 +1,218 @@
+INFO 2026-07-17 15:43:27 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-17 15:43:27 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/sz25gl12
+INFO 2026-07-17 15:43:27 ot_train.py:298 Creating dataset
+INFO 2026-07-17 15:43:31 ot_train.py:332 Creating policy
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 1175.90it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1943.98it/s]
+INFO 2026-07-17 15:43:36 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-17 15:43:36 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K
+INFO 2026-07-17 15:43:36 ot_train.py:446 cfg.steps=60000 (60K)
+INFO 2026-07-17 15:43:36 ot_train.py:447 dataset.num_frames=218367 (218K)
+INFO 2026-07-17 15:43:36 ot_train.py:448 dataset.num_episodes=4930
+INFO 2026-07-17 15:43:36 ot_train.py:451 Effective batch size: 128 x 1 = 128
+INFO 2026-07-17 15:43:36 ot_train.py:452 num_learnable_params=1622090881 (2B)
+INFO 2026-07-17 15:43:36 ot_train.py:453 num_total_params=3145712033 (3B)
+Training:   0%|          | 0/60000 [00:00<?, ?step/s]INFO 2026-07-17 15:43:36 ot_train.py:604 Start offline training on a fixed dataset, with effective batch size: 128
+Training:   0%|          | 200/60000 [17:46<57:11:40,  3.44s/step]INFO 2026-07-17 16:01:22 ot_train.py:649 step:200 smpl:26K ep:578 epch:0.12 loss:1.233 grdn:0.465 lr:1.7e-06 updt_s:1.774 data_s:3.552 smp/s:24 mem_gb:52.89
+Training:   1%|          | 400/60000 [29:13<54:37:54,  3.30s/step]INFO 2026-07-17 16:12:49 ot_train.py:649 step:400 smpl:51K ep:1K epch:0.23 loss:0.859 grdn:0.963 lr:5.0e-06 updt_s:1.780 data_s:1.649 smp/s:37 mem_gb:52.95
+Training:   1%|          | 600/60000 [40:39<61:05:01,  3.70s/step]INFO 2026-07-17 16:24:15 ot_train.py:649 step:600 smpl:77K ep:2K epch:0.35 loss:0.336 grdn:1.622 lr:8.3e-06 updt_s:1.787 data_s:1.636 smp/s:37 mem_gb:52.95
+Training:   1%|▏         | 800/60000 [51:55<53:43:10,  3.27s/step]INFO 2026-07-17 16:35:31 ot_train.py:649 step:800 smpl:102K ep:2K epch:0.47 loss:0.212 grdn:2.051 lr:1.2e-05 updt_s:1.774 data_s:1.601 smp/s:38 mem_gb:52.95
+Training:   2%|▏         | 1000/60000 [1:03:15<51:44:23,  3.16s/step]INFO 2026-07-17 16:46:51 ot_train.py:649 step:1K smpl:128K ep:3K epch:0.59 loss:0.173 grdn:2.122 lr:1.5e-05 updt_s:1.807 data_s:1.590 smp/s:38 mem_gb:52.95
+Training:   2%|▏         | 1200/60000 [1:14:32<61:06:37,  3.74s/step]INFO 2026-07-17 16:58:08 ot_train.py:649 step:1K smpl:154K ep:3K epch:0.70 loss:0.151 grdn:2.006 lr:1.8e-05 updt_s:1.813 data_s:1.567 smp/s:38 mem_gb:52.88
+Training:   2%|▏         | 1400/60000 [1:25:28<58:59:01,  3.62s/step]INFO 2026-07-17 17:09:05 ot_train.py:649 step:1K smpl:179K ep:4K epch:0.82 loss:0.136 grdn:1.855 lr:2.2e-05 updt_s:1.776 data_s:1.500 smp/s:39 mem_gb:52.98
+Training:   3%|▎         | 1600/60000 [1:36:15<51:24:54,  3.17s/step]INFO 2026-07-17 17:19:51 ot_train.py:649 step:2K smpl:205K ep:5K epch:0.94 loss:0.125 grdn:1.691 lr:2.5e-05 updt_s:1.773 data_s:1.454 smp/s:40 mem_gb:52.98
+Training:   3%|▎         | 1800/60000 [1:46:54<50:53:11,  3.15s/step]INFO 2026-07-17 17:30:30 ot_train.py:649 step:2K smpl:230K ep:5K epch:1.06 loss:0.118 grdn:1.530 lr:2.8e-05 updt_s:1.733 data_s:1.460 smp/s:40 mem_gb:52.98
+Training:   3%|▎         | 2000/60000 [1:57:30<46:27:57,  2.88s/step]INFO 2026-07-17 17:41:06 ot_train.py:649 step:2K smpl:256K ep:6K epch:1.17 loss:0.111 grdn:1.401 lr:3.2e-05 updt_s:1.780 data_s:1.396 smp/s:40 mem_gb:52.98
+Training:   4%|▎         | 2200/60000 [2:08:00<50:46:50,  3.16s/step]INFO 2026-07-17 17:51:36 ot_train.py:649 step:2K smpl:282K ep:6K epch:1.29 loss:0.105 grdn:1.245 lr:3.5e-05 updt_s:1.726 data_s:1.418 smp/s:41 mem_gb:52.98
+Training:   4%|▍         | 2400/60000 [2:18:23<50:59:59,  3.19s/step]INFO 2026-07-17 18:01:59 ot_train.py:649 step:2K smpl:307K ep:7K epch:1.41 loss:0.100 grdn:1.116 lr:3.8e-05 updt_s:1.703 data_s:1.406 smp/s:41 mem_gb:52.90
+Training:   4%|▍         | 2600/60000 [2:29:57<52:04:27,  3.27s/step]INFO 2026-07-17 18:13:33 ot_train.py:649 step:3K smpl:333K ep:8K epch:1.52 loss:0.096 grdn:1.001 lr:4.2e-05 updt_s:1.737 data_s:1.727 smp/s:37 mem_gb:52.98
+Training:   5%|▍         | 2800/60000 [2:40:46<48:19:29,  3.04s/step]INFO 2026-07-17 18:24:22 ot_train.py:649 step:3K smpl:358K ep:8K epch:1.64 loss:0.091 grdn:0.939 lr:4.5e-05 updt_s:1.772 data_s:1.470 smp/s:39 mem_gb:52.98
+Training:   5%|▌         | 3000/60000 [2:51:38<51:35:36,  3.26s/step]INFO 2026-07-17 18:35:14 ot_train.py:649 step:3K smpl:384K ep:9K epch:1.76 loss:0.087 grdn:0.832 lr:4.8e-05 updt_s:1.766 data_s:1.487 smp/s:39 mem_gb:52.98
+Training:   5%|▌         | 3200/60000 [3:02:51<51:44:14,  3.28s/step]INFO 2026-07-17 18:46:27 ot_train.py:649 step:3K smpl:410K ep:9K epch:1.88 loss:0.083 grdn:0.773 lr:5.2e-05 updt_s:1.703 data_s:1.657 smp/s:38 mem_gb:52.98
+Training:   6%|▌         | 3400/60000 [3:13:36<50:03:54,  3.18s/step]INFO 2026-07-17 18:57:12 ot_train.py:649 step:3K smpl:435K ep:10K epch:1.99 loss:0.081 grdn:0.742 lr:5.5e-05 updt_s:1.678 data_s:1.544 smp/s:40 mem_gb:52.90
+Training:   6%|▌         | 3600/60000 [3:24:36<52:33:54,  3.36s/step]INFO 2026-07-17 19:08:12 ot_train.py:649 step:4K smpl:461K ep:10K epch:2.11 loss:0.077 grdn:0.684 lr:5.8e-05 updt_s:1.674 data_s:1.620 smp/s:39 mem_gb:52.98
+Training:   6%|▋         | 3800/60000 [3:35:47<49:35:34,  3.18s/step]INFO 2026-07-17 19:19:23 ot_train.py:649 step:4K smpl:486K ep:11K epch:2.23 loss:0.074 grdn:0.639 lr:6.2e-05 updt_s:1.620 data_s:1.733 smp/s:38 mem_gb:52.98
+Training:   7%|▋         | 4000/60000 [3:47:37<58:10:28,  3.74s/step]INFO 2026-07-17 19:31:13 ot_train.py:649 step:4K smpl:512K ep:12K epch:2.34 loss:0.072 grdn:0.614 lr:6.5e-05 updt_s:1.678 data_s:1.865 smp/s:36 mem_gb:52.98
+Training:   7%|▋         | 4200/60000 [4:00:12<54:10:31,  3.50s/step]INFO 2026-07-17 19:43:48 ot_train.py:649 step:4K smpl:538K ep:12K epch:2.46 loss:0.070 grdn:0.567 lr:6.8e-05 updt_s:1.619 data_s:2.151 smp/s:34 mem_gb:52.98
+Training:   7%|▋         | 4400/60000 [4:12:06<53:47:33,  3.48s/step]INFO 2026-07-17 19:55:42 ot_train.py:649 step:4K smpl:563K ep:13K epch:2.58 loss:0.068 grdn:0.549 lr:7.2e-05 updt_s:1.680 data_s:1.886 smp/s:36 mem_gb:52.98
+Training:   8%|▊         | 4600/60000 [4:23:59<53:08:05,  3.45s/step]INFO 2026-07-17 20:07:35 ot_train.py:649 step:5K smpl:589K ep:13K epch:2.70 loss:0.067 grdn:0.518 lr:7.5e-05 updt_s:1.736 data_s:1.825 smp/s:36 mem_gb:52.90
+Training:   8%|▊         | 4800/60000 [4:35:32<47:32:41,  3.10s/step]INFO 2026-07-17 20:19:08 ot_train.py:649 step:5K smpl:614K ep:14K epch:2.81 loss:0.064 grdn:0.520 lr:7.8e-05 updt_s:1.801 data_s:1.659 smp/s:37 mem_gb:52.98
+Training:   8%|▊         | 5000/60000 [4:46:16<48:21:38,  3.17s/step]INFO 2026-07-17 20:29:52 ot_train.py:649 step:5K smpl:640K ep:14K epch:2.93 loss:0.063 grdn:0.483 lr:8.2e-05 updt_s:1.752 data_s:1.464 smp/s:40 mem_gb:52.98
+Training:   9%|▊         | 5200/60000 [4:57:13<50:14:42,  3.30s/step]INFO 2026-07-17 20:40:49 ot_train.py:649 step:5K smpl:666K ep:15K epch:3.05 loss:0.061 grdn:0.458 lr:8.5e-05 updt_s:1.756 data_s:1.525 smp/s:39 mem_gb:52.98
+Training:   9%|▉         | 5400/60000 [5:09:24<58:35:42,  3.86s/step]INFO 2026-07-17 20:53:00 ot_train.py:649 step:5K smpl:691K ep:16K epch:3.17 loss:0.059 grdn:0.434 lr:8.8e-05 updt_s:1.694 data_s:1.953 smp/s:35 mem_gb:52.98
+Training:   9%|▉         | 5600/60000 [5:20:27<46:25:15,  3.07s/step]INFO 2026-07-17 21:04:03 ot_train.py:649 step:6K smpl:717K ep:16K epch:3.28 loss:0.058 grdn:0.423 lr:9.2e-05 updt_s:1.640 data_s:1.671 smp/s:39 mem_gb:52.98
+Training:  10%|▉         | 5800/60000 [5:31:10<50:29:01,  3.35s/step]INFO 2026-07-17 21:14:46 ot_train.py:649 step:6K smpl:742K ep:17K epch:3.40 loss:0.057 grdn:0.414 lr:9.5e-05 updt_s:1.632 data_s:1.575 smp/s:40 mem_gb:52.90
+Training:  10%|█         | 6000/60000 [5:41:54<48:49:09,  3.25s/step]INFO 2026-07-17 21:25:30 ot_train.py:649 step:6K smpl:768K ep:17K epch:3.52 loss:0.057 grdn:0.406 lr:9.8e-05 updt_s:1.605 data_s:1.611 smp/s:40 mem_gb:52.98
+INFO 2026-07-17 21:25:30 ot_train.py:103 INSIGHT phase transition: index=1 name=fm_only range=[6000,60000) trainable_params=1620515968
+Training:  10%|█         | 6200/60000 [5:52:29<46:16:41,  3.10s/step]INFO 2026-07-17 21:36:05 ot_train.py:649 step:6K smpl:794K ep:18K epch:3.63 loss:0.056 grdn:0.392 lr:1.0e-04 updt_s:1.586 data_s:1.583 smp/s:40 mem_gb:52.85
+Training:  11%|█         | 6400/60000 [6:03:06<46:02:29,  3.09s/step]INFO 2026-07-17 21:46:42 ot_train.py:649 step:6K smpl:819K ep:18K epch:3.75 loss:0.054 grdn:0.365 lr:1.0e-04 updt_s:1.605 data_s:1.572 smp/s:40 mem_gb:52.85
+Training:  11%|█         | 6600/60000 [6:13:43<47:06:11,  3.18s/step]INFO 2026-07-17 21:57:19 ot_train.py:649 step:7K smpl:845K ep:19K epch:3.87 loss:0.054 grdn:0.357 lr:1.0e-04 updt_s:1.610 data_s:1.571 smp/s:40 mem_gb:52.85
+Training:  11%|█▏        | 6800/60000 [6:24:09<46:44:09,  3.16s/step]INFO 2026-07-17 22:07:45 ot_train.py:649 step:7K smpl:870K ep:20K epch:3.99 loss:0.053 grdn:0.349 lr:1.0e-04 updt_s:1.552 data_s:1.570 smp/s:41 mem_gb:52.77
+Training:  12%|█▏        | 7000/60000 [6:34:31<43:13:43,  2.94s/step]INFO 2026-07-17 22:18:07 ot_train.py:649 step:7K smpl:896K ep:20K epch:4.10 loss:0.052 grdn:0.340 lr:1.0e-04 updt_s:1.640 data_s:1.466 smp/s:41 mem_gb:52.85
+Training:  12%|█▏        | 7200/60000 [6:44:48<44:04:39,  3.01s/step]INFO 2026-07-17 22:28:25 ot_train.py:649 step:7K smpl:922K ep:21K epch:4.22 loss:0.051 grdn:0.331 lr:1.0e-04 updt_s:1.618 data_s:1.466 smp/s:42 mem_gb:52.85
+Training:  12%|█▏        | 7400/60000 [6:55:03<44:10:37,  3.02s/step]INFO 2026-07-17 22:38:40 ot_train.py:649 step:7K smpl:947K ep:21K epch:4.34 loss:0.052 grdn:0.348 lr:1.0e-04 updt_s:1.626 data_s:1.444 smp/s:42 mem_gb:52.85
+Training:  13%|█▎        | 7600/60000 [7:05:22<45:24:06,  3.12s/step]INFO 2026-07-17 22:48:58 ot_train.py:649 step:8K smpl:973K ep:22K epch:4.45 loss:0.052 grdn:0.342 lr:1.0e-04 updt_s:1.606 data_s:1.480 smp/s:41 mem_gb:52.85
+Training:  13%|█���        | 7800/60000 [7:15:48<45:55:41,  3.17s/step]INFO 2026-07-17 22:59:24 ot_train.py:649 step:8K smpl:998K ep:23K epch:4.57 loss:0.051 grdn:0.314 lr:1.0e-04 updt_s:1.608 data_s:1.517 smp/s:41 mem_gb:52.85
+Training:  13%|█▎        | 8000/60000 [7:26:09<51:11:15,  3.54s/step]INFO 2026-07-17 23:09:45 ot_train.py:649 step:8K smpl:1M ep:23K epch:4.69 loss:0.049 grdn:0.313 lr:1.0e-04 updt_s:1.568 data_s:1.530 smp/s:41 mem_gb:52.77
+Training:  14%|█▎        | 8200/60000 [7:36:30<45:59:57,  3.20s/step]INFO 2026-07-17 23:20:06 ot_train.py:649 step:8K smpl:1M ep:24K epch:4.81 loss:0.048 grdn:0.300 lr:1.0e-04 updt_s:1.586 data_s:1.515 smp/s:41 mem_gb:52.85
+Training:  14%|█▍        | 8400/60000 [7:46:48<44:33:07,  3.11s/step]INFO 2026-07-17 23:30:24 ot_train.py:649 step:8K smpl:1M ep:24K epch:4.92 loss:0.048 grdn:0.307 lr:1.0e-04 updt_s:1.581 data_s:1.501 smp/s:42 mem_gb:52.85
+Training:  14%|█▍        | 8600/60000 [7:57:14<44:40:43,  3.13s/step]INFO 2026-07-17 23:40:50 ot_train.py:649 step:9K smpl:1M ep:25K epch:5.04 loss:0.047 grdn:0.300 lr:9.9e-05 updt_s:1.645 data_s:1.479 smp/s:41 mem_gb:52.85
+Training:  15%|█▍        | 8800/60000 [8:07:35<43:40:53,  3.07s/step]INFO 2026-07-17 23:51:11 ot_train.py:649 step:9K smpl:1M ep:25K epch:5.16 loss:0.047 grdn:0.291 lr:9.9e-05 updt_s:1.594 data_s:1.506 smp/s:41 mem_gb:52.85
+Training:  15%|█▌        | 9000/60000 [8:17:59<44:14:47,  3.12s/step]INFO 2026-07-18 00:01:35 ot_train.py:649 step:9K smpl:1M ep:26K epch:5.28 loss:0.050 grdn:0.314 lr:9.9e-05 updt_s:1.553 data_s:1.562 smp/s:41 mem_gb:52.85
+Training:  15%|█▌        | 9200/60000 [8:28:26<44:07:30,  3.13s/step]INFO 2026-07-18 00:12:03 ot_train.py:649 step:9K smpl:1M ep:27K epch:5.39 loss:0.045 grdn:0.276 lr:9.9e-05 updt_s:1.596 data_s:1.536 smp/s:41 mem_gb:52.77
+Training:  16%|█▌        | 9400/60000 [8:38:51<42:34:53,  3.03s/step]INFO 2026-07-18 00:22:27 ot_train.py:649 step:9K smpl:1M ep:27K epch:5.51 loss:0.046 grdn:0.298 lr:9.9e-05 updt_s:1.617 data_s:1.499 smp/s:41 mem_gb:52.85
+Training:  16%|█▌        | 9600/60000 [8:49:15<43:35:40,  3.11s/step]INFO 2026-07-18 00:32:52 ot_train.py:649 step:10K smpl:1M ep:28K epch:5.63 loss:0.046 grdn:0.285 lr:9.9e-05 updt_s:1.633 data_s:1.484 smp/s:41 mem_gb:52.85
+Training:  16%|█▋        | 9800/60000 [8:59:25<42:32:33,  3.05s/step]INFO 2026-07-18 00:43:02 ot_train.py:649 step:10K smpl:1M ep:28K epch:5.74 loss:0.045 grdn:0.279 lr:9.9e-05 updt_s:1.608 data_s:1.437 smp/s:42 mem_gb:52.85
+Training:  17%|█▋        | 10000/60000 [9:09:29<39:33:39,  2.85s/step]INFO 2026-07-18 00:53:05 ot_train.py:649 step:10K smpl:1M ep:29K epch:5.86 loss:0.045 grdn:0.266 lr:9.9e-05 updt_s:1.615 data_s:1.396 smp/s:43 mem_gb:52.85
+Training:  17%|█▋        | 10200/60000 [9:19:37<45:29:41,  3.29s/step]INFO 2026-07-18 01:03:13 ot_train.py:649 step:10K smpl:1M ep:29K epch:5.98 loss:0.045 grdn:0.268 lr:9.9e-05 updt_s:1.639 data_s:1.394 smp/s:42 mem_gb:52.77
+Training:  17%|█▋        | 10400/60000 [9:29:49<41:05:39,  2.98s/step]INFO 2026-07-18 01:13:25 ot_train.py:649 step:10K smpl:1M ep:30K epch:6.10 loss:0.044 grdn:0.266 lr:9.8e-05 updt_s:1.660 data_s:1.399 smp/s:42 mem_gb:52.85
+Training:  18%|█▊        | 10600/60000 [9:39:57<43:06:12,  3.14s/step]INFO 2026-07-18 01:23:33 ot_train.py:649 step:11K smpl:1M ep:31K epch:6.21 loss:0.044 grdn:0.274 lr:9.8e-05 updt_s:1.666 data_s:1.369 smp/s:42 mem_gb:52.85
+Training:  18%|█▊        | 10800/60000 [9:50:14<46:36:08,  3.41s/step]INFO 2026-07-18 01:33:50 ot_train.py:649 step:11K smpl:1M ep:31K epch:6.33 loss:0.045 grdn:0.272 lr:9.8e-05 updt_s:1.588 data_s:1.489 smp/s:42 mem_gb:52.85
+Training:  18%|█▊        | 11000/60000 [10:00:32<42:20:18,  3.11s/step]INFO 2026-07-18 01:44:09 ot_train.py:649 step:11K smpl:1M ep:32K epch:6.45 loss:0.043 grdn:0.261 lr:9.8e-05 updt_s:1.576 data_s:1.512 smp/s:41 mem_gb:52.85
+Training:  19%|█▊        | 11200/60000 [10:10:42<43:28:02,  3.21s/step]INFO 2026-07-18 01:54:18 ot_train.py:649 step:11K smpl:1M ep:32K epch:6.57 loss:0.044 grdn:0.259 lr:9.8e-05 updt_s:1.615 data_s:1.428 smp/s:42 mem_gb:52.85
+Training:  19%|█▉        | 11400/60000 [10:20:58<46:57:06,  3.48s/step]INFO 2026-07-18 02:04:35 ot_train.py:649 step:11K smpl:1M ep:33K epch:6.68 loss:0.042 grdn:0.268 lr:9.8e-05 updt_s:1.640 data_s:1.436 smp/s:42 mem_gb:52.77
+Training:  19%|█▉        | 11600/60000 [10:31:06<40:11:20,  2.99s/step]INFO 2026-07-18 02:14:42 ot_train.py:649 step:12K smpl:1M ep:34K epch:6.80 loss:0.042 grdn:0.253 lr:9.7e-05 updt_s:1.604 data_s:1.427 smp/s:42 mem_gb:52.85
+Training:  20%|█▉        | 11800/60000 [10:41:23<41:45:51,  3.12s/step]INFO 2026-07-18 02:25:00 ot_train.py:649 step:12K smpl:2M ep:34K epch:6.92 loss:0.041 grdn:0.252 lr:9.7e-05 updt_s:1.577 data_s:1.506 smp/s:42 mem_gb:52.85
+Training:  20%|██        | 12000/60000 [10:51:45<41:52:17,  3.14s/step]INFO 2026-07-18 02:35:21 ot_train.py:649 step:12K smpl:2M ep:35K epch:7.03 loss:0.042 grdn:0.257 lr:9.7e-05 updt_s:1.517 data_s:1.587 smp/s:41 mem_gb:52.85
+Training:  20%|██        | 12200/60000 [11:02:00<41:48:14,  3.15s/step]INFO 2026-07-18 02:45:36 ot_train.py:649 step:12K smpl:2M ep:35K epch:7.15 loss:0.041 grdn:0.250 lr:9.7e-05 updt_s:1.586 data_s:1.481 smp/s:42 mem_gb:52.85
+Training:  21%|██        | 12400/60000 [11:12:04<41:00:36,  3.10s/step]INFO 2026-07-18 02:55:40 ot_train.py:649 step:12K smpl:2M ep:36K epch:7.27 loss:0.041 grdn:0.248 lr:9.7e-05 updt_s:1.780 data_s:1.237 smp/s:42 mem_gb:52.85
+Training:  21%|██        | 12600/60000 [11:22:06<39:20:54,  2.99s/step]INFO 2026-07-18 03:05:42 ot_train.py:649 step:13K smpl:2M ep:36K epch:7.39 loss:0.040 grdn:0.246 lr:9.6e-05 updt_s:1.808 data_s:1.196 smp/s:43 mem_gb:52.77
+Training:  21%|██▏       | 12800/60000 [11:32:04<39:12:32,  2.99s/step]INFO 2026-07-18 03:15:40 ot_train.py:649 step:13K smpl:2M ep:37K epch:7.50 loss:0.040 grdn:0.241 lr:9.6e-05 updt_s:1.785 data_s:1.199 smp/s:43 mem_gb:52.85
+Training:  22%|██▏       | 13000/60000 [11:42:07<37:37:38,  2.88s/step]INFO 2026-07-18 03:25:43 ot_train.py:649 step:13K smpl:2M ep:38K epch:7.62 loss:0.040 grdn:0.244 lr:9.6e-05 updt_s:1.784 data_s:1.226 smp/s:43 mem_gb:52.85
+Training:  22%|██▏       | 13200/60000 [11:52:02<39:58:36,  3.08s/step]INFO 2026-07-18 03:35:38 ot_train.py:649 step:13K smpl:2M ep:38K epch:7.74 loss:0.039 grdn:0.241 lr:9.6e-05 updt_s:1.753 data_s:1.217 smp/s:43 mem_gb:52.85
+Training:  22%|██▏       | 13400/60000 [12:01:54<37:13:39,  2.88s/step]INFO 2026-07-18 03:45:31 ot_train.py:649 step:13K smpl:2M ep:39K epch:7.85 loss:0.039 grdn:0.247 lr:9.6e-05 updt_s:1.744 data_s:1.215 smp/s:43 mem_gb:52.85
+Training:  23%|██▎       | 13600/60000 [12:11:58<39:35:23,  3.07s/step]INFO 2026-07-18 03:55:34 ot_train.py:649 step:14K smpl:2M ep:39K epch:7.97 loss:0.038 grdn:0.228 lr:9.5e-05 updt_s:1.779 data_s:1.235 smp/s:42 mem_gb:52.77
+Training:  23%|██▎       | 13800/60000 [12:22:00<39:56:20,  3.11s/step]INFO 2026-07-18 04:05:36 ot_train.py:649 step:14K smpl:2M ep:40K epch:8.09 loss:0.037 grdn:0.239 lr:9.5e-05 updt_s:1.788 data_s:1.217 smp/s:43 mem_gb:52.85
+Training:  23%|██▎       | 14000/60000 [12:32:06<36:37:45,  2.87s/step]INFO 2026-07-18 04:15:43 ot_train.py:649 step:14K smpl:2M ep:40K epch:8.21 loss:0.037 grdn:0.237 lr:9.5e-05 updt_s:1.760 data_s:1.268 smp/s:42 mem_gb:52.85
+Training:  24%|██▎       | 14200/60000 [12:42:15<35:55:22,  2.82s/step]INFO 2026-07-18 04:25:51 ot_train.py:649 step:14K smpl:2M ep:41K epch:8.32 loss:0.039 grdn:0.240 lr:9.5e-05 updt_s:1.738 data_s:1.300 smp/s:42 mem_gb:52.85
+Training:  24%|██▍       | 14400/60000 [12:52:19<37:39:36,  2.97s/step]INFO 2026-07-18 04:35:55 ot_train.py:649 step:14K smpl:2M ep:42K epch:8.44 loss:0.041 grdn:0.267 lr:9.4e-05 updt_s:1.735 data_s:1.282 smp/s:42 mem_gb:52.85
+Training:  24%|██▍       | 14600/60000 [13:02:24<37:31:44,  2.98s/step]INFO 2026-07-18 04:46:00 ot_train.py:649 step:15K smpl:2M ep:42K epch:8.56 loss:0.038 grdn:0.242 lr:9.4e-05 updt_s:1.771 data_s:1.250 smp/s:42 mem_gb:52.85
+Training:  25%|██▍       | 14800/60000 [13:12:26<37:31:13,  2.99s/step]INFO 2026-07-18 04:56:03 ot_train.py:649 step:15K smpl:2M ep:43K epch:8.68 loss:0.037 grdn:0.237 lr:9.4e-05 updt_s:1.727 data_s:1.280 smp/s:43 mem_gb:52.77
+Training:  25%|██▌       | 15000/60000 [13:22:35<39:05:06,  3.13s/step]INFO 2026-07-18 05:06:11 ot_train.py:649 step:15K smpl:2M ep:43K epch:8.79 loss:0.037 grdn:0.237 lr:9.3e-05 updt_s:1.814 data_s:1.225 smp/s:42 mem_gb:52.85
+Training:  25%|██▌       | 15200/60000 [13:32:53<38:56:20,  3.13s/step]INFO 2026-07-18 05:16:29 ot_train.py:649 step:15K smpl:2M ep:44K epch:8.91 loss:0.037 grdn:0.230 lr:9.3e-05 updt_s:1.748 data_s:1.335 smp/s:42 mem_gb:52.85
+Training:  26%|██▌       | 15400/60000 [13:43:11<36:18:46,  2.93s/step]INFO 2026-07-18 05:26:47 ot_train.py:649 step:15K smpl:2M ep:45K epch:9.03 loss:0.037 grdn:0.229 lr:9.3e-05 updt_s:1.615 data_s:1.472 smp/s:41 mem_gb:52.85
+Training:  26%|██▌       | 15600/60000 [13:53:30<37:33:18,  3.05s/step]INFO 2026-07-18 05:37:06 ot_train.py:649 step:16K smpl:2M ep:45K epch:9.14 loss:0.035 grdn:0.223 lr:9.3e-05 updt_s:1.582 data_s:1.505 smp/s:41 mem_gb:52.85
+Training:  26%|██▋       | 15800/60000 [14:03:50<37:32:14,  3.06s/step]INFO 2026-07-18 05:47:26 ot_train.py:649 step:16K smpl:2M ep:46K epch:9.26 loss:0.036 grdn:0.225 lr:9.2e-05 updt_s:1.586 data_s:1.510 smp/s:41 mem_gb:52.85
+Training:  27%|██▋       | 16000/60000 [14:14:09<37:20:22,  3.06s/step]INFO 2026-07-18 05:57:45 ot_train.py:649 step:16K smpl:2M ep:46K epch:9.38 loss:0.034 grdn:0.225 lr:9.2e-05 updt_s:1.580 data_s:1.509 smp/s:41 mem_gb:52.77
+Training:  27%|██▋       | 16200/60000 [14:24:26<38:41:45,  3.18s/step]INFO 2026-07-18 06:08:02 ot_train.py:649 step:16K smpl:2M ep:47K epch:9.50 loss:0.035 grdn:0.229 lr:9.2e-05 updt_s:1.573 data_s:1.510 smp/s:42 mem_gb:52.85
+Training:  27%|██▋       | 16400/60000 [14:34:41<37:48:53,  3.12s/step]INFO 2026-07-18 06:18:17 ot_train.py:649 step:16K smpl:2M ep:47K epch:9.61 loss:0.035 grdn:0.227 lr:9.1e-05 updt_s:1.577 data_s:1.492 smp/s:42 mem_gb:52.85
+Training:  28%|██▊       | 16600/60000 [14:45:03<37:13:50,  3.09s/step]INFO 2026-07-18 06:28:40 ot_train.py:649 step:17K smpl:2M ep:48K epch:9.73 loss:0.035 grdn:0.220 lr:9.1e-05 updt_s:1.583 data_s:1.523 smp/s:41 mem_gb:52.85
+Training:  28%|██▊       | 16800/60000 [14:55:19<38:22:40,  3.20s/step]INFO 2026-07-18 06:38:55 ot_train.py:649 step:17K smpl:2M ep:49K epch:9.85 loss:0.035 grdn:0.222 lr:9.1e-05 updt_s:1.537 data_s:1.534 smp/s:42 mem_gb:52.85
+Training:  28%|██▊       | 17000/60000 [15:05:32<36:47:49,  3.08s/step]INFO 2026-07-18 06:49:08 ot_train.py:649 step:17K smpl:2M ep:49K epch:9.96 loss:0.036 grdn:0.234 lr:9.0e-05 updt_s:1.583 data_s:1.477 smp/s:42 mem_gb:52.77
+Training:  29%|██▊       | 17200/60000 [15:16:20<44:41:29,  3.76s/step]INFO 2026-07-18 06:59:56 ot_train.py:649 step:17K smpl:2M ep:50K epch:10.08 loss:0.033 grdn:0.224 lr:9.0e-05 updt_s:1.657 data_s:1.577 smp/s:40 mem_gb:52.85
+Training:  29%|██▉       | 17400/60000 [15:26:57<37:16:33,  3.15s/step]INFO 2026-07-18 07:10:33 ot_train.py:649 step:17K smpl:2M ep:50K epch:10.20 loss:0.033 grdn:0.224 lr:9.0e-05 updt_s:1.607 data_s:1.576 smp/s:40 mem_gb:52.85
+Training:  29%|██▉       | 17600/60000 [15:37:32<35:39:29,  3.03s/step]INFO 2026-07-18 07:21:08 ot_train.py:649 step:18K smpl:2M ep:51K epch:10.32 loss:0.033 grdn:0.207 lr:8.9e-05 updt_s:1.590 data_s:1.580 smp/s:40 mem_gb:52.85
+Training:  30%|██▉       | 17800/60000 [15:48:13<36:24:02,  3.11s/step]INFO 2026-07-18 07:31:49 ot_train.py:649 step:18K smpl:2M ep:51K epch:10.43 loss:0.033 grdn:0.221 lr:8.9e-05 updt_s:1.630 data_s:1.569 smp/s:40 mem_gb:52.85
+Training:  30%|███       | 18000/60000 [15:58:52<39:11:44,  3.36s/step]INFO 2026-07-18 07:42:29 ot_train.py:649 step:18K smpl:2M ep:52K epch:10.55 loss:0.033 grdn:0.224 lr:8.8e-05 updt_s:1.622 data_s:1.573 smp/s:40 mem_gb:52.85
+Training:  30%|███       | 18200/60000 [16:09:33<37:27:13,  3.23s/step]INFO 2026-07-18 07:53:09 ot_train.py:649 step:18K smpl:2M ep:53K epch:10.67 loss:0.034 grdn:0.221 lr:8.8e-05 updt_s:1.610 data_s:1.588 smp/s:40 mem_gb:52.77
+Training:  31%|███       | 18400/60000 [16:20:24<36:01:54,  3.12s/step]INFO 2026-07-18 08:04:00 ot_train.py:649 step:18K smpl:2M ep:53K epch:10.79 loss:0.033 grdn:0.229 lr:8.8e-05 updt_s:1.648 data_s:1.601 smp/s:39 mem_gb:52.85
+Training:  31%|███       | 18600/60000 [16:30:51<35:56:39,  3.13s/step]INFO 2026-07-18 08:14:27 ot_train.py:649 step:19K smpl:2M ep:54K epch:10.90 loss:0.032 grdn:0.211 lr:8.7e-05 updt_s:1.541 data_s:1.591 smp/s:41 mem_gb:52.85
+Training:  31%|███▏      | 18800/60000 [16:41:06<36:03:46,  3.15s/step]INFO 2026-07-18 08:24:42 ot_train.py:649 step:19K smpl:2M ep:54K epch:11.02 loss:0.032 grdn:0.210 lr:8.7e-05 updt_s:1.538 data_s:1.532 smp/s:42 mem_gb:52.85
+Training:  32%|███▏      | 19000/60000 [16:51:44<40:26:02,  3.55s/step]INFO 2026-07-18 08:35:20 ot_train.py:649 step:19K smpl:2M ep:55K epch:11.14 loss:0.031 grdn:0.205 lr:8.7e-05 updt_s:1.623 data_s:1.561 smp/s:40 mem_gb:52.85
+Training:  32%|███▏      | 19200/60000 [17:02:16<36:02:44,  3.18s/step]INFO 2026-07-18 08:45:52 ot_train.py:649 step:19K smpl:2M ep:55K epch:11.25 loss:0.033 grdn:0.226 lr:8.6e-05 updt_s:1.583 data_s:1.575 smp/s:41 mem_gb:52.77
+Training:  32%|███▏      | 19400/60000 [17:12:44<31:35:48,  2.80s/step]INFO 2026-07-18 08:56:21 ot_train.py:649 step:19K smpl:2M ep:56K epch:11.37 loss:0.031 grdn:0.224 lr:8.6e-05 updt_s:1.591 data_s:1.546 smp/s:41 mem_gb:52.85
+Training:  33%|███▎      | 19600/60000 [17:22:38<35:12:48,  3.14s/step]INFO 2026-07-18 09:06:14 ot_train.py:649 step:20K smpl:3M ep:57K epch:11.49 loss:0.031 grdn:0.207 lr:8.5e-05 updt_s:1.633 data_s:1.329 smp/s:43 mem_gb:52.85
+Training:  33%|███▎      | 19800/60000 [17:32:21<30:56:45,  2.77s/step]INFO 2026-07-18 09:15:57 ot_train.py:649 step:20K smpl:3M ep:57K epch:11.61 loss:0.030 grdn:0.211 lr:8.5e-05 updt_s:1.603 data_s:1.304 smp/s:44 mem_gb:52.85
+Training:  33%|███▎      | 20000/60000 [17:42:10<32:17:24,  2.91s/step]INFO 2026-07-18 09:25:46 ot_train.py:649 step:20K smpl:3M ep:58K epch:11.72 loss:0.031 grdn:0.209 lr:8.5e-05 updt_s:1.604 data_s:1.336 smp/s:44 mem_gb:52.85
+Training:  34%|███▎      | 20200/60000 [17:51:57<30:58:34,  2.80s/step]INFO 2026-07-18 09:35:33 ot_train.py:649 step:20K smpl:3M ep:58K epch:11.84 loss:0.030 grdn:0.215 lr:8.4e-05 updt_s:1.624 data_s:1.307 smp/s:44 mem_gb:52.85
+Training:  34%|███▍      | 20400/60000 [18:01:49<33:31:13,  3.05s/step]INFO 2026-07-18 09:45:25 ot_train.py:649 step:20K smpl:3M ep:59K epch:11.96 loss:0.030 grdn:0.208 lr:8.4e-05 updt_s:1.624 data_s:1.329 smp/s:43 mem_gb:52.77
+Training:  34%|███▍      | 20600/60000 [18:11:53<31:59:43,  2.92s/step]INFO 2026-07-18 09:55:29 ot_train.py:649 step:21K smpl:3M ep:60K epch:12.08 loss:0.029 grdn:0.200 lr:8.3e-05 updt_s:1.652 data_s:1.363 smp/s:42 mem_gb:52.85
+Training:  35%|███▍      | 20800/60000 [18:21:47<30:39:15,  2.82s/step]INFO 2026-07-18 10:05:23 ot_train.py:649 step:21K smpl:3M ep:60K epch:12.19 loss:0.029 grdn:0.208 lr:8.3e-05 updt_s:1.633 data_s:1.334 smp/s:43 mem_gb:52.85
+Training:  35%|███▌      | 21000/60000 [18:31:43<33:55:54,  3.13s/step]INFO 2026-07-18 10:15:19 ot_train.py:649 step:21K smpl:3M ep:61K epch:12.31 loss:0.029 grdn:0.202 lr:8.2e-05 updt_s:1.639 data_s:1.335 smp/s:43 mem_gb:52.85
+Training:  35%|███▌      | 21200/60000 [18:41:34<31:44:25,  2.94s/step]INFO 2026-07-18 10:25:11 ot_train.py:649 step:21K smpl:3M ep:61K epch:12.43 loss:0.030 grdn:0.208 lr:8.2e-05 updt_s:1.616 data_s:1.336 smp/s:43 mem_gb:52.85
+Training:  36%|███▌      | 21400/60000 [18:51:24<31:42:04,  2.96s/step]INFO 2026-07-18 10:35:00 ot_train.py:649 step:21K smpl:3M ep:62K epch:12.54 loss:0.029 grdn:0.200 lr:8.1e-05 updt_s:1.584 data_s:1.359 smp/s:43 mem_gb:52.85
+Training:  36%|███▌      | 21600/60000 [19:00:59<31:09:33,  2.92s/step]INFO 2026-07-18 10:44:35 ot_train.py:649 step:22K smpl:3M ep:62K epch:12.66 loss:0.030 grdn:0.222 lr:8.1e-05 updt_s:1.582 data_s:1.286 smp/s:45 mem_gb:52.77
+Training:  36%|███▋      | 21800/60000 [19:10:29<33:28:47,  3.16s/step]INFO 2026-07-18 10:54:06 ot_train.py:649 step:22K smpl:3M ep:63K epch:12.78 loss:0.028 grdn:0.211 lr:8.1e-05 updt_s:1.574 data_s:1.275 smp/s:45 mem_gb:52.85
+Training:  37%|███▋      | 22000/60000 [19:20:02<30:10:33,  2.86s/step]INFO 2026-07-18 11:03:38 ot_train.py:649 step:22K smpl:3M ep:64K epch:12.90 loss:0.028 grdn:0.204 lr:8.0e-05 updt_s:1.591 data_s:1.269 smp/s:45 mem_gb:52.85
+Training:  37%|███▋      | 22200/60000 [19:29:28<31:03:57,  2.96s/step]INFO 2026-07-18 11:13:04 ot_train.py:649 step:22K smpl:3M ep:64K epch:13.01 loss:0.029 grdn:0.210 lr:8.0e-05 updt_s:1.536 data_s:1.288 smp/s:45 mem_gb:52.85
+Training:  37%|███▋      | 22400/60000 [19:38:57<28:50:38,  2.76s/step]INFO 2026-07-18 11:22:33 ot_train.py:649 step:22K smpl:3M ep:65K epch:13.13 loss:0.027 grdn:0.194 lr:7.9e-05 updt_s:1.556 data_s:1.284 smp/s:45 mem_gb:52.85
+Training:  38%|███▊      | 22600/60000 [19:49:05<31:44:04,  3.05s/step]INFO 2026-07-18 11:32:41 ot_train.py:649 step:23K smpl:3M ep:65K epch:13.25 loss:0.028 grdn:0.205 lr:7.9e-05 updt_s:1.743 data_s:1.294 smp/s:42 mem_gb:52.77
+Training:  38%|███▊      | 22800/60000 [19:59:25<32:26:54,  3.14s/step]INFO 2026-07-18 11:43:01 ot_train.py:649 step:23K smpl:3M ep:66K epch:13.36 loss:0.027 grdn:0.202 lr:7.8e-05 updt_s:1.811 data_s:1.285 smp/s:41 mem_gb:52.85
+Training:  38%|███▊      | 23000/60000 [20:09:40<31:42:25,  3.09s/step]INFO 2026-07-18 11:53:16 ot_train.py:649 step:23K smpl:3M ep:66K epch:13.48 loss:0.027 grdn:0.203 lr:7.8e-05 updt_s:1.762 data_s:1.305 smp/s:42 mem_gb:52.85
+Training:  39%|███▊      | 23200/60000 [20:19:53<31:15:38,  3.06s/step]INFO 2026-07-18 12:03:30 ot_train.py:649 step:23K smpl:3M ep:67K epch:13.60 loss:0.027 grdn:0.195 lr:7.7e-05 updt_s:1.804 data_s:1.260 smp/s:42 mem_gb:52.85
+Training:  39%|███▉      | 23400/60000 [20:29:57<31:13:50,  3.07s/step]INFO 2026-07-18 12:13:34 ot_train.py:649 step:23K smpl:3M ep:68K epch:13.72 loss:0.026 grdn:0.199 lr:7.7e-05 updt_s:1.764 data_s:1.251 smp/s:42 mem_gb:52.85
+Training:  39%|███▉      | 23600/60000 [20:40:09<31:42:13,  3.14s/step]INFO 2026-07-18 12:23:46 ot_train.py:649 step:24K smpl:3M ep:68K epch:13.83 loss:0.026 grdn:0.198 lr:7.6e-05 updt_s:1.784 data_s:1.272 smp/s:42 mem_gb:52.85
+Training:  40%|███▉      | 23800/60000 [20:50:20<30:35:00,  3.04s/step]INFO 2026-07-18 12:33:57 ot_train.py:649 step:24K smpl:3M ep:69K epch:13.95 loss:0.026 grdn:0.193 lr:7.6e-05 updt_s:1.778 data_s:1.272 smp/s:42 mem_gb:52.77
+Training:  40%|████      | 24000/60000 [21:00:34<31:31:05,  3.15s/step]INFO 2026-07-18 12:44:10 ot_train.py:649 step:24K smpl:3M ep:69K epch:14.07 loss:0.025 grdn:0.195 lr:7.5e-05 updt_s:1.735 data_s:1.326 smp/s:42 mem_gb:52.85
+Training:  40%|████      | 24200/60000 [21:11:01<30:26:56,  3.06s/step]INFO 2026-07-18 12:54:37 ot_train.py:649 step:24K smpl:3M ep:70K epch:14.19 loss:0.025 grdn:0.191 lr:7.5e-05 updt_s:1.560 data_s:1.570 smp/s:41 mem_gb:52.85
+Training:  41%|████      | 24400/60000 [21:21:15<30:34:01,  3.09s/step]INFO 2026-07-18 13:04:51 ot_train.py:649 step:24K smpl:3M ep:71K epch:14.30 loss:0.025 grdn:0.188 lr:7.4e-05 updt_s:1.808 data_s:1.258 smp/s:42 mem_gb:52.85
+Training:  41%|████      | 24600/60000 [21:31:36<31:01:47,  3.16s/step]INFO 2026-07-18 13:15:12 ot_train.py:649 step:25K smpl:3M ep:71K epch:14.42 loss:0.027 grdn:0.208 lr:7.4e-05 updt_s:1.733 data_s:1.366 smp/s:41 mem_gb:52.85
+Training:  41%|████▏     | 24800/60000 [21:42:05<30:13:56,  3.09s/step]INFO 2026-07-18 13:25:41 ot_train.py:649 step:25K smpl:3M ep:72K epch:14.54 loss:0.026 grdn:0.197 lr:7.3e-05 updt_s:1.554 data_s:1.586 smp/s:41 mem_gb:52.85
+Training:  42%|████▏     | 25000/60000 [21:52:31<30:16:53,  3.11s/step]INFO 2026-07-18 13:36:07 ot_train.py:649 step:25K smpl:3M ep:72K epch:14.65 loss:0.024 grdn:0.187 lr:7.3e-05 updt_s:1.586 data_s:1.539 smp/s:41 mem_gb:52.77
+Training:  42%|████▏     | 25200/60000 [22:02:54<29:34:37,  3.06s/step]INFO 2026-07-18 13:46:30 ot_train.py:649 step:25K smpl:3M ep:73K epch:14.77 loss:0.024 grdn:0.190 lr:7.2e-05 updt_s:1.650 data_s:1.458 smp/s:41 mem_gb:52.85
+Training:  42%|████▏     | 25400/60000 [22:13:12<28:09:50,  2.93s/step]INFO 2026-07-18 13:56:48 ot_train.py:649 step:25K smpl:3M ep:73K epch:14.89 loss:0.025 grdn:0.193 lr:7.2e-05 updt_s:1.623 data_s:1.464 smp/s:41 mem_gb:52.85
+Training:  43%|████▎     | 25600/60000 [22:23:19<29:49:19,  3.12s/step]INFO 2026-07-18 14:06:55 ot_train.py:649 step:26K smpl:3M ep:74K epch:15.01 loss:0.026 grdn:0.200 lr:7.1e-05 updt_s:1.582 data_s:1.444 smp/s:42 mem_gb:52.85
+Training:  43%|████▎     | 25800/60000 [22:33:35<27:45:28,  2.92s/step]INFO 2026-07-18 14:17:11 ot_train.py:649 step:26K smpl:3M ep:75K epch:15.12 loss:0.024 grdn:0.185 lr:7.1e-05 updt_s:1.649 data_s:1.426 smp/s:42 mem_gb:52.85
+Training:  43%|████▎     | 26000/60000 [22:43:57<30:43:07,  3.25s/step]INFO 2026-07-18 14:27:33 ot_train.py:649 step:26K smpl:3M ep:75K epch:15.24 loss:0.024 grdn:0.185 lr:7.0e-05 updt_s:1.673 data_s:1.434 smp/s:41 mem_gb:52.77
+Training:  44%|████▎     | 26200/60000 [22:54:09<29:37:02,  3.15s/step]INFO 2026-07-18 14:37:46 ot_train.py:649 step:26K smpl:3M ep:76K epch:15.36 loss:0.023 grdn:0.185 lr:7.0e-05 updt_s:1.590 data_s:1.467 smp/s:42 mem_gb:52.85
+Training:  44%|████▍     | 26400/60000 [23:04:18<28:10:49,  3.02s/step]INFO 2026-07-18 14:47:54 ot_train.py:649 step:26K smpl:3M ep:76K epch:15.47 loss:0.023 grdn:0.187 lr:6.9e-05 updt_s:1.549 data_s:1.489 smp/s:42 mem_gb:52.85
+Training:  44%|████▍     | 26600/60000 [23:14:43<28:29:42,  3.07s/step]INFO 2026-07-18 14:58:19 ot_train.py:649 step:27K smpl:3M ep:77K epch:15.59 loss:0.022 grdn:0.179 lr:6.8e-05 updt_s:1.564 data_s:1.554 smp/s:41 mem_gb:52.85
+Training:  45%|████▍     | 26800/60000 [23:25:00<30:34:26,  3.32s/step]INFO 2026-07-18 15:08:36 ot_train.py:649 step:27K smpl:3M ep:77K epch:15.71 loss:0.023 grdn:0.181 lr:6.8e-05 updt_s:1.570 data_s:1.509 smp/s:42 mem_gb:52.85
+Training:  45%|████▌     | 27000/60000 [23:35:14<27:21:48,  2.99s/step]INFO 2026-07-18 15:18:50 ot_train.py:649 step:27K smpl:3M ep:78K epch:15.83 loss:0.023 grdn:0.185 lr:6.7e-05 updt_s:1.567 data_s:1.501 smp/s:42 mem_gb:52.85
+Training:  45%|████▌     | 27200/60000 [23:45:38<27:56:54,  3.07s/step]INFO 2026-07-18 15:29:15 ot_train.py:649 step:27K smpl:3M ep:79K epch:15.94 loss:0.022 grdn:0.173 lr:6.7e-05 updt_s:1.606 data_s:1.509 smp/s:41 mem_gb:52.77
+Training:  46%|████▌     | 27400/60000 [23:55:46<27:19:33,  3.02s/step]INFO 2026-07-18 15:39:23 ot_train.py:649 step:27K smpl:4M ep:79K epch:16.06 loss:0.022 grdn:0.179 lr:6.6e-05 updt_s:1.612 data_s:1.423 smp/s:42 mem_gb:52.85
+Training:  46%|████▌     | 27600/60000 [24:05:54<27:44:18,  3.08s/step]INFO 2026-07-18 15:49:30 ot_train.py:649 step:28K smpl:4M ep:80K epch:16.18 loss:0.022 grdn:0.175 lr:6.6e-05 updt_s:1.612 data_s:1.419 smp/s:42 mem_gb:52.85
+Training:  46%|████▋     | 27800/60000 [24:16:04<27:02:33,  3.02s/step]INFO 2026-07-18 15:59:40 ot_train.py:649 step:28K smpl:4M ep:80K epch:16.30 loss:0.023 grdn:0.183 lr:6.5e-05 updt_s:1.670 data_s:1.376 smp/s:42 mem_gb:52.85
+Training:  47%|████▋     | 28000/60000 [24:25:52<25:10:49,  2.83s/step]INFO 2026-07-18 16:09:28 ot_train.py:649 step:28K smpl:4M ep:81K epch:16.41 loss:0.023 grdn:0.186 lr:6.5e-05 updt_s:1.594 data_s:1.339 smp/s:44 mem_gb:52.85
+Training:  47%|████▋     | 28200/60000 [24:36:03<26:43:19,  3.03s/step]INFO 2026-07-18 16:19:39 ot_train.py:649 step:28K smpl:4M ep:81K epch:16.53 loss:0.022 grdn:0.174 lr:6.4e-05 updt_s:1.649 data_s:1.403 smp/s:42 mem_gb:52.85
+Training:  47%|████▋     | 28400/60000 [24:46:18<27:00:46,  3.08s/step]INFO 2026-07-18 16:29:54 ot_train.py:649 step:28K smpl:4M ep:82K epch:16.65 loss:0.020 grdn:0.170 lr:6.4e-05 updt_s:1.564 data_s:1.504 smp/s:42 mem_gb:52.77
+Training:  48%|████▊     | 28600/60000 [24:56:28<25:58:21,  2.98s/step]INFO 2026-07-18 16:40:05 ot_train.py:649 step:29K smpl:4M ep:83K epch:16.76 loss:0.021 grdn:0.183 lr:6.3e-05 updt_s:1.589 data_s:1.458 smp/s:42 mem_gb:52.85
+Training:  48%|████▊     | 28800/60000 [25:06:32<27:14:29,  3.14s/step]INFO 2026-07-18 16:50:08 ot_train.py:649 step:29K smpl:4M ep:83K epch:16.88 loss:0.022 grdn:0.185 lr:6.2e-05 updt_s:1.637 data_s:1.376 smp/s:42 mem_gb:52.85
+Training:  48%|████▊     | 29000/60000 [25:16:50<25:42:42,  2.99s/step]INFO 2026-07-18 17:00:26 ot_train.py:649 step:29K smpl:4M ep:84K epch:17.00 loss:0.021 grdn:0.174 lr:6.2e-05 updt_s:1.614 data_s:1.468 smp/s:42 mem_gb:52.85
+Training:  49%|████▊     | 29200/60000 [25:27:06<26:22:50,  3.08s/step]INFO 2026-07-18 17:10:42 ot_train.py:649 step:29K smpl:4M ep:84K epch:17.12 loss:0.021 grdn:0.177 lr:6.1e-05 updt_s:1.583 data_s:1.492 smp/s:42 mem_gb:52.85
+Training:  49%|████▉     | 29400/60000 [25:37:14<26:39:06,  3.14s/step]INFO 2026-07-18 17:20:51 ot_train.py:649 step:29K smpl:4M ep:85K epch:17.23 loss:0.021 grdn:0.176 lr:6.1e-05 updt_s:1.605 data_s:1.433 smp/s:42 mem_gb:52.77
+Training:  49%|████▉     | 29600/60000 [25:47:26<25:24:04,  3.01s/step]INFO 2026-07-18 17:31:02 ot_train.py:649 step:30K smpl:4M ep:86K epch:17.35 loss:0.020 grdn:0.178 lr:6.0e-05 updt_s:1.600 data_s:1.451 smp/s:42 mem_gb:52.85
+Training:  50%|████▉     | 29800/60000 [25:57:33<25:21:40,  3.02s/step]INFO 2026-07-18 17:41:09 ot_train.py:649 step:30K smpl:4M ep:86K epch:17.47 loss:0.020 grdn:0.173 lr:6.0e-05 updt_s:1.685 data_s:1.342 smp/s:42 mem_gb:52.85
+Training:  50%|█████     | 30000/60000 [26:07:28<24:39:55,  2.96s/step]INFO 2026-07-18 17:51:04 ot_train.py:649 step:30K smpl:4M ep:87K epch:17.59 loss:0.020 grdn:0.171 lr:5.9e-05 updt_s:1.640 data_s:1.333 smp/s:43 mem_gb:52.85
+Training:  50%|█████     | 30200/60000 [26:17:37<24:15:16,  2.93s/step]INFO 2026-07-18 18:01:13 ot_train.py:649 step:30K smpl:4M ep:87K epch:17.70 loss:0.020 grdn:0.163 lr:5.8e-05 updt_s:1.603 data_s:1.436 smp/s:42 mem_gb:52.85
+Training:  51%|█████     | 30400/60000 [26:27:46<25:17:59,  3.08s/step]INFO 2026-07-18 18:11:22 ot_train.py:649 step:30K smpl:4M ep:88K epch:17.82 loss:0.020 grdn:0.166 lr:5.8e-05 updt_s:1.630 data_s:1.412 smp/s:42 mem_gb:52.85
+Training:  51%|█████     | 30600/60000 [26:38:00<23:53:26,  2.93s/step]INFO 2026-07-18 18:21:36 ot_train.py:649 step:31K smpl:4M ep:88K epch:17.94 loss:0.019 grdn:0.164 lr:5.7e-05 updt_s:1.576 data_s:1.489 smp/s:42 mem_gb:52.77
+Training:  51%|█████▏    | 30800/60000 [26:48:09<24:05:11,  2.97s/step]INFO 2026-07-18 18:31:45 ot_train.py:649 step:31K smpl:4M ep:89K epch:18.05 loss:0.018 grdn:0.160 lr:5.7e-05 updt_s:1.624 data_s:1.416 smp/s:42 mem_gb:52.85
+Training:  52%|█████▏    | 31000/60000 [26:58:15<24:23:44,  3.03s/step]INFO 2026-07-18 18:41:51 ot_train.py:649 step:31K smpl:4M ep:90K epch:18.17 loss:0.020 grdn:0.170 lr:5.6e-05 updt_s:1.608 data_s:1.414 smp/s:42 mem_gb:52.85
+Training:  52%|█████▏    | 31200/60000 [27:08:33<23:36:26,  2.95s/step]INFO 2026-07-18 18:52:09 ot_train.py:649 step:31K smpl:4M ep:90K epch:18.29 loss:0.019 grdn:0.165 lr:5.6e-05 updt_s:1.672 data_s:1.415 smp/s:41 mem_gb:52.85
+Training:  52%|█████▏    | 31400/60000 [27:18:34<22:09:17,  2.79s/step]INFO 2026-07-18 19:02:10 ot_train.py:649 step:31K smpl:4M ep:91K epch:18.41 loss:0.019 grdn:0.164 lr:5.5e-05 updt_s:1.596 data_s:1.402 smp/s:43 mem_gb:52.85
+Training:  53%|█████▎    | 31600/60000 [27:28:43<24:50:40,  3.15s/step]INFO 2026-07-18 19:12:19 ot_train.py:649 step:32K smpl:4M ep:91K epch:18.52 loss:0.019 grdn:0.167 lr:5.4e-05 updt_s:1.624 data_s:1.416 smp/s:42 mem_gb:52.85
+Training:  53%|█████▎    | 31800/60000 [27:38:36<26:20:54,  3.36s/step]INFO 2026-07-18 19:22:12 ot_train.py:649 step:32K smpl:4M ep:92K epch:18.64 loss:0.018 grdn:0.168 lr:5.4e-05 updt_s:1.610 data_s:1.350 smp/s:43 mem_gb:52.77
+Training:  53%|█████▎    | 32000/60000 [27:48:42<23:44:24,  3.05s/step]INFO 2026-07-18 19:32:18 ot_train.py:649 step:32K smpl:4M ep:92K epch:18.76 loss:0.018 grdn:0.180 lr:5.3e-05 updt_s:1.595 data_s:1.431 smp/s:42 mem_gb:52.85
+Training:  54%|█████▎    | 32200/60000 [27:58:40<26:04:51,  3.38s/step]INFO 2026-07-18 19:42:16 ot_train.py:649 step:32K smpl:4M ep:93K epch:18.87 loss:0.019 grdn:0.171 lr:5.3e-05 updt_s:1.586 data_s:1.397 smp/s:43 mem_gb:52.85
+Training:  54%|█████▍    | 32400/60000 [28:08:34<21:34:22,  2.81s/step]INFO 2026-07-18 19:52:10 ot_train.py:649 step:32K smpl:4M ep:94K epch:18.99 loss:0.018 grdn:0.162 lr:5.2e-05 updt_s:1.614 data_s:1.351 smp/s:43 mem_gb:52.85
+Training:  54%|█████▍    | 32600/60000 [28:18:33<22:50:50,  3.00s/step]INFO 2026-07-18 20:02:09 ot_train.py:649 step:33K smpl:4M ep:94K epch:19.11 loss:0.018 grdn:0.170 lr:5.1e-05 updt_s:1.616 data_s:1.376 smp/s:43 mem_gb:52.85
+Training:  55%|█████▍    | 32800/60000 [28:28:41<23:37:20,  3.13s/step]INFO 2026-07-18 20:12:17 ot_train.py:649 step:33K smpl:4M ep:95K epch:19.23 loss:0.018 grdn:0.166 lr:5.1e-05 updt_s:1.644 data_s:1.388 smp/s:42 mem_gb:52.77
+Training:  55%|█████▌    | 33000/60000 [28:38:43<22:09:47,  2.96s/step]INFO 2026-07-18 20:22:19 ot_train.py:649 step:33K smpl:4M ep:95K epch:19.34 loss:0.017 grdn:0.150 lr:5.0e-05 updt_s:1.623 data_s:1.382 smp/s:43 mem_gb:52.85
+Training:  55%|█████▌    | 33200/60000 [28:48:45<24:19:20,  3.27s/step]INFO 2026-07-18 20:32:21 ot_train.py:649 step:33K smpl:4M ep:96K epch:19.46 loss:0.017 grdn:0.165 lr:5.0e-05 updt_s:1.617 data_s:1.391 smp/s:43 mem_gb:52.85
+Training:  56%|█████▌    | 33400/60000 [28:58:58<23:03:55,  3.12s/step]INFO 2026-07-18 20:42:34 ot_train.py:649 step:33K smpl:4M ep:97K epch:19.58 loss:0.017 grdn:0.150 lr:4.9e-05 updt_s:1.607 data_s:1.451 smp/s:42 mem_gb:52.85
+Training:  56%|█████▌    | 33600/60000 [29:09:11<23:05:53,  3.15s/step]INFO 2026-07-18 20:52:47 ot_train.py:649 step:34K smpl:4M ep:97K epch:19.70 loss:0.017 grdn:0.157 lr:4.9e-05 updt_s:1.619 data_s:1.439 smp/s:42 mem_gb:52.85
+Training:  56%|█████▋    | 33800/60000 [29:19:25<23:22:37,  3.21s/step]INFO 2026-07-18 21:03:01 ot_train.py:649 step:34K smpl:4M ep:98K epch:19.81 loss:0.017 grdn:0.151 lr:4.8e-05 updt_s:1.617 data_s:1.448 smp/s:42 mem_gb:52.85
+Training:  57%|█████▋    | 34000/60000 [29:29:37<21:52:02,  3.03s/step]INFO 2026-07-18 21:13:13 ot_train.py:649 step:34K smpl:4M ep:98K epch:19.93 loss:0.016 grdn:0.153 lr:4.7e-05 updt_s:1.582 data_s:1.474 smp/s:42 mem_gb:52.77
+Training:  57%|█████▋    | 34200/60000 [29:39:49<19:11:36,  2.68s/step]INFO 2026-07-18 21:23:26 ot_train.py:649 step:34K smpl:4M ep:99K epch:20.05 loss:0.016 grdn:0.156 lr:4.7e-05 updt_s:1.564 data_s:1.495 smp/s:42 mem_gb:52.85
+Training:  57%|█████▋    | 34400/60000 [29:49:22<19:36:52,  2.76s/step]INFO 2026-07-18 21:32:58 ot_train.py:649 step:34K smpl:4M ep:99K epch:20.16 loss:0.016 grdn:0.157 lr:4.6e-05 updt_s:1.608 data_s:1.250 smp/s:45 mem_gb:52.85
+Training:  58%|█████▊    | 34600/60000 [29:58:47<19:36:45,  2.78s/step]INFO 2026-07-18 21:42:23 ot_train.py:649 step:35K smpl:4M ep:100K epch:20.28 loss:0.017 grdn:0.165 lr:4.6e-05 updt_s:1.588 data_s:1.232 smp/s:45 mem_gb:52.85
+Training:  58%|█████▊    | 34800/60000 [30:08:20<20:17:25,  2.90s/step]INFO 2026-07-18 21:51:56 ot_train.py:649 step:35K smpl:4M ep:101K epch:20.40 loss:0.016 grdn:0.149 lr:4.5e-05 updt_s:1.602 data_s:1.257 smp/s:45 mem_gb:52.85
+Training:  58%|█████▊    | 35000/60000 [30:17:43<22:50:19,  3.29s/step]INFO 2026-07-18 22:01:19 ot_train.py:649 step:35K smpl:4M ep:101K epch:20.52 loss:0.016 grdn:0.156 lr:4.4e-05 updt_s:1.547 data_s:1.263 smp/s:46 mem_gb:52.77
+Training:  59%|█████▊    | 35200/60000 [30:27:09<17:43:14,  2.57s/step]INFO 2026-07-18 22:10:45 ot_train.py:649 step:35K smpl:5M ep:102K epch:20.63 loss:0.015 grdn:0.150 lr:4.4e-05 updt_s:1.594 data_s:1.230 smp/s:45 mem_gb:52.85
+Training:  59%|█████▉    | 35400/60000 [30:36:38<19:18:31,  2.83s/step]INFO 2026-07-18 22:20:14 ot_train.py:649 step:35K smpl:5M ep:102K epch:20.75 loss:0.015 grdn:0.139 lr:4.3e-05 updt_s:1.587 data_s:1.255 smp/s:45 mem_gb:52.85
+Training:  59%|█████▉    | 35600/60000 [30:46:05<18:27:03,  2.72s/step]INFO 2026-07-18 22:29:41 ot_train.py:649 step:36K smpl:5M ep:103K epch:20.87 loss:0.015 grdn:0.149 lr:4.3e-05 updt_s:1.637 data_s:1.190 smp/s:45 mem_gb:52.85
+Training:  60%|█████▉    | 35800/60000 [30:55:28<17:46:57,  2.65s/step]INFO 2026-07-18 22:39:04 ot_train.py:649 step:36K smpl:5M ep:103K epch:20.98 loss:0.015 grdn:0.143 lr:4.2e-05 updt_s:1.592 data_s:1.221 smp/s:45 mem_gb:52.85
+Training:  60%|██████    | 36000/60000 [31:04:46<19:31:07,  2.93s/step]INFO 2026-07-18 22:48:22 ot_train.py:649 step:36K smpl:5M ep:104K epch:21.10 loss:0.015 grdn:0.144 lr:4.2e-05 updt_s:1.578 data_s:1.205 smp/s:46 mem_gb:52.85
+Training:  60%|██████    | 36200/60000 [31:14:48<20:23:12,  3.08s/step]INFO 2026-07-18 22:58:24 ot_train.py:649 step:36K smpl:5M ep:105K epch:21.22 loss:0.015 grdn:0.157 lr:4.1e-05 updt_s:1.611 data_s:1.396 smp/s:43 mem_gb:52.77
+Training:  61%|██████    | 36400/60000 [31:24:40<20:51:24,  3.18s/step]INFO 2026-07-18 23:08:16 ot_train.py:649 step:36K smpl:5M ep:105K epch:21.34 loss:0.014 grdn:0.141 lr:4.0e-05 updt_s:1.556 data_s:1.399 smp/s:43 mem_gb:52.85
+Training:  61%|██████    | 36600/60000 [31:35:05<24:13:28,  3.73s/step]INFO 2026-07-18 23:18:41 ot_train.py:649 step:37K smpl:5M ep:106K epch:21.45 loss:0.015 grdn:0.150 lr:4.0e-05 updt_s:1.659 data_s:1.459 smp/s:41 mem_gb:52.85
+Training:  61%|██████▏   | 36800/60000 [31:45:51<22:01:31,  3.42s/step]INFO 2026-07-18 23:29:27 ot_train.py:649 step:37K smpl:5M ep:106K epch:21.57 loss:0.015 grdn:0.144 lr:3.9e-05 updt_s:1.639 data_s:1.585 smp/s:40 mem_gb:52.85
+Training:  62%|██████▏   | 37000/60000 [31:56:35<20:17:38,  3.18s/step]INFO 2026-07-18 23:40:11 ot_train.py:649 step:37K smpl:5M ep:107K epch:21.69 loss:0.015 grdn:0.149 lr:3.9e-05 updt_s:1.582 data_s:1.634 smp/s:40 mem_gb:52.85
+Training:  62%|██████▏   | 37200/60000 [32:07:14<20:12:06,  3.19s/step]INFO 2026-07-18 23:50:51 ot_train.py:649 step:37K smpl:5M ep:108K epch:21.81 loss:0.014 grdn:0.140 lr:3.8e-05 updt_s:1.578 data_s:1.612 smp/s:40 mem_gb:52.85
+Training:  62%|██████▏   | 37400/60000 [32:17:58<19:34:44,  3.12s/step]INFO 2026-07-19 00:01:35 ot_train.py:649 step:37K smpl:5M ep:108K epch:21.92 loss:0.014 grdn:0.135 lr:3.8e-05 updt_s:1.622 data_s:1.592 smp/s:40 mem_gb:52.77
+Training:  63%|██████▎   | 37600/60000 [32:28:15<19:32:49,  3.14s/step]INFO 2026-07-19 00:11:52 ot_train.py:649 step:38K smpl:5M ep:109K epch:22.04 loss:0.014 grdn:0.133 lr:3.7e-05 updt_s:1.525 data_s:1.554 smp/s:42 mem_gb:52.85
+Training:  63%|██████▎   | 37800/60000 [32:38:31<18:54:05,  3.07s/step]INFO 2026-07-19 00:22:07 ot_train.py:649 step:38K smpl:5M ep:109K epch:22.16 loss:0.014 grdn:0.139 lr:3.6e-05 updt_s:1.601 data_s:1.472 smp/s:42 mem_gb:52.85
+Training:  63%|██████▎   | 38000/60000 [32:48:48<18:30:21,  3.03s/step]INFO 2026-07-19 00:32:24 ot_train.py:649 step:38K smpl:5M ep:110K epch:22.27 loss:0.013 grdn:0.135 lr:3.6e-05 updt_s:1.610 data_s:1.470 smp/s:42 mem_gb:52.85
+Training:  64%|██████▎   | 38200/60000 [32:59:03<18:26:15,  3.04s/step]INFO 2026-07-19 00:42:39 ot_train.py:649 step:38K smpl:5M ep:110K epch:22.39 loss:0.013 grdn:0.133 lr:3.5e-05 updt_s:1.560 data_s:1.506 smp/s:42 mem_gb:52.85
+Training:  64%|██████▍   | 38400/60000 [33:09:08<18:26:14,  3.07s/step]INFO 2026-07-19 00:52:44 ot_train.py:649 step:38K smpl:5M ep:111K epch:22.51 loss:0.014 grdn:0.146 lr:3.5e-05 updt_s:1.596 data_s:1.426 smp/s:42 mem_gb:52.77
+Training:  64%|██████▍   | 38600/60000 [33:19:24<17:44:30,  2.98s/step]INFO 2026-07-19 01:03:00 ot_train.py:649 step:39K smpl:5M ep:112K epch:22.63 loss:0.013 grdn:0.129 lr:3.4e-05 updt_s:1.649 data_s:1.424 smp/s:42 mem_gb:52.85
+Training:  65%|██████▍   | 38800/60000 [33:29:38<17:53:09,  3.04s/step]INFO 2026-07-19 01:13:14 ot_train.py:649 step:39K smpl:5M ep:112K epch:22.74 loss:0.013 grdn:0.141 lr:3.4e-05 updt_s:1.646 data_s:1.422 smp/s:42 mem_gb:52.85
+Training:  65%|██████▌   | 39000/60000 [33:39:46<17:57:58,  3.08s/step]INFO 2026-07-19 01:23:23 ot_train.py:649 step:39K smpl:5M ep:113K epch:22.86 loss:0.013 grdn:0.133 lr:3.3e-05 updt_s:1.633 data_s:1.403 smp/s:42 mem_gb:52.85
+Training:  65%|██████▌   | 39200/60000 [33:49:57<18:14:43,  3.16s/step]INFO 2026-07-19 01:33:33 ot_train.py:649 step:39K smpl:5M ep:113K epch:22.98 loss:0.013 grdn:0.133 lr:3.3e-05 updt_s:1.589 data_s:1.456 smp/s:42 mem_gb:52.85
+Training:  66%|██████▌   | 39400/60000 [34:00:19<16:01:33,  2.80s/step]INFO 2026-07-19 01:43:56 ot_train.py:649 step:39K smpl:5M ep:114K epch:23.10 loss:0.013 grdn:0.139 lr:3.2e-05 updt_s:1.587 data_s:1.522 smp/s:41 mem_gb:52.85
+Training:  66%|██████▌   | 39600/60000 [34:10:03<15:31:18,  2.74s/step]INFO 2026-07-19 01:53:39 ot_train.py:649 step:40K smpl:5M ep:114K epch:23.21 loss:0.013 grdn:0.129 lr:3.2e-05 updt_s:1.506 data_s:1.407 smp/s:44 mem_gb:52.77
+Training:  66%|██████▋   | 39800/60000 [34:19:50<15:45:11,  2.81s/step]INFO 2026-07-19 02:03:26 ot_train.py:649 step:40K smpl:5M ep:115K epch:23.33 loss:0.012 grdn:0.141 lr:3.1e-05 updt_s:1.520 data_s:1.409 smp/s:44 mem_gb:52.85
+Training:  67%|██████▋   | 40000/60000 [34:29:21<17:31:13,  3.15s/step]INFO 2026-07-19 02:12:57 ot_train.py:649 step:40K smpl:5M ep:116K epch:23.45 loss:0.012 grdn:0.134 lr:3.0e-05 updt_s:1.489 data_s:1.359 smp/s:45 mem_gb:52.85
+Training:  67%|██████▋   | 40111/60000 [34:34:56<16:35:21,  3.00s/step]

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/LAPstyle_linear_6K.interrupted_reboot_20260719T051924/wandb/run-20260717_154326-sz25gl12/files/requirements.txt

@@ -0,0 +1,117 @@
+pyarrow==25.0.0
+importlib_metadata==9.0.0
+wheel==0.47.0
+cloudpickle==3.1.2
+nvidia-cusolver-cu12==11.7.3.90
+multiprocess==0.70.19
+torch==2.11.0+cu128
+nvidia-nvjitlink-cu12==12.8.93
+fsspec==2026.2.0
+pytz==2026.2
+hf-xet==1.5.1
+tokenizers==0.22.2
+opencv-python-headless==4.13.0.92
+cuda-pathfinder==1.2.2
+frozenlist==1.8.0
+smmap==5.0.3
+httpcore==1.0.9
+nvidia-nvshmem-cu12==3.4.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+annotated-types==0.7.0
+platformdirs==4.10.0
+click==8.4.2
+pyyaml-include==1.4.1
+MarkupSafe==3.0.3
+Jinja2==3.1.6
+urllib3==2.7.0
+nvidia-curand-cu12==10.3.9.90
+certifi==2026.6.17
+nvidia-cuda-runtime-cu12==12.8.90
+torchcodec==0.11.1
+toml==0.10.2
+av==15.1.0
+nvidia-cufft-cu12==11.3.3.83
+attrs==26.1.0
+lerobot==0.6.0
+PyYAML==6.0.3
+safetensors==0.8.0
+diffusers==0.35.2
+requests==2.34.2
+nvidia-nvtx-cu12==12.8.90
+charset-normalizer==3.4.9
+networkx==3.6.1
+cmake==4.1.3
+pydantic_core==2.46.4
+timm==1.0.28
+dm-tree==0.1.10
+nvidia-cufile-cu12==1.13.1.3
+termcolor==3.3.0
+multidict==6.7.1
+filelock==3.29.0
+tzdata==2026.3
+aiohappyeyeballs==2.7.1
+httpx==0.28.1
+nvidia-nccl-cu12==2.28.9
+Pygments==2.20.0
+einops==0.8.2
+nvidia-cusparse-cu12==12.5.8.93
+torchvision==0.26.0+cu128
+dill==0.4.1
+GitPython==3.1.51
+nvidia-cuda-cupti-cu12==12.8.90
+decord==0.6.0
+sympy==1.14.0
+cuda-toolkit==12.8.1
+python-dateutil==2.9.0.post0
+shellingham==1.5.4
+propcache==0.5.2
+Farama-Notifications==0.0.6
+pillow==12.2.0
+cuda-bindings==12.9.4
+huggingface_hub==1.23.0
+pandas==2.3.3
+typing-inspect==0.9.0
+packaging==25.0
+rich==15.0.0
+nvidia-cusparselt-cu12==0.7.1
+setuptools==78.1.0
+wrapt==2.2.2
+mypy_extensions==1.1.0
+tqdm==4.68.4
+draccus==0.10.0
+aiosignal==1.4.0
+mpmath==1.3.0
+numpy==2.2.6
+typing_extensions==4.15.0
+psutil==7.2.2
+mdurl==0.1.2
+nvidia-cublas-cu12==12.8.4.1
+absl-py==2.5.0
+gitdb==4.0.12
+transformers==5.5.4
+accelerate==1.14.0
+pydantic==2.13.4
+pip==26.1.2
+markdown-it-py==4.2.0
+typer==0.26.8
+triton==3.6.0
+wandb==0.27.2
+regex==2026.7.10
+yarl==1.24.2
+jsonlines==4.0.0
+mergedeep==1.3.4
+aiohttp==3.14.1
+datasets==4.8.5
+gymnasium==1.3.0
+peft==0.19.1
+zipp==4.1.0
+idna==3.18
+nvidia-cudnn-cu12==9.19.0.56
+h11==0.16.0
+sentry-sdk==2.65.0
+annotated-doc==0.0.4
+protobuf==7.35.1
+typing-inspection==0.4.2
+anyio==4.14.2
+xxhash==3.8.1
+six==1.17.0

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/LAPstyle_linear_6K.interrupted_reboot_20260719T051924/wandb/run-20260717_154326-sz25gl12/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+  "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-17T06:43:26.118344Z",
+  "args": [
+    "--config_path=/tmp/insight_lerobot_cfg_yp0hihif/LAPstyle_RKD_linear_decay_6K.yaml",
+    "--batch_size=128"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "codePathLocal": "my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a970d9f98f189faccce13cf2fb61ccf5d55921f3"
+  },
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K",
+  "host": "DGX-H200-01",
+  "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "1493416677376"
+    }
+  },
+  "memory": {
+    "total": "2164177248256"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "18e0dkydbeh9kbgur2u0ri0dyzmcya11"
+}

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/LAPstyle_linear_6K.interrupted_reboot_20260719T051924/wandb/run-20260717_154326-sz25gl12/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_setup.py:_flush():81] Configure stats pid to 499778
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-17 15:43:26,119 INFO    MainThread:499778 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260717_154326-sz25gl12/logs/debug.log
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260717_154326-sz25gl12/logs/debug-internal.log
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:init():766] calling init triggers
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-17 15:43:26,120 INFO    MainThread:499778 [wandb_init.py:init():814] starting backend
+2026-07-17 15:43:26,346 INFO    MainThread:499778 [wandb_init.py:init():829] sending inform_init request
+2026-07-17 15:43:26,990 INFO    MainThread:499778 [wandb_init.py:init():834] backend started and connected
+2026-07-17 15:43:26,993 INFO    MainThread:499778 [wandb_init.py:init():904] updated telemetry
+2026-07-17 15:43:27,003 INFO    MainThread:499778 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-17 15:43:27,554 INFO    MainThread:499778 [wandb_init.py:init():972] starting run threads in backend
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-17 15:43:27,750 INFO    MainThread:499778 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-17 15:43:27,754 INFO    MainThread:499778 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/RSCLstyle_cosine_60K.interrupted_reboot_20260719T051924/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_setup.py:_flush():81] Configure stats pid to 499773
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260717_154331-e5avxg6w/logs/debug.log
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260717_154331-e5avxg6w/logs/debug-internal.log
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:init():766] calling init triggers
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:init():814] starting backend
+2026-07-17 15:43:31,678 INFO    MainThread:499773 [wandb_init.py:init():829] sending inform_init request
+2026-07-17 15:43:32,380 INFO    MainThread:499773 [wandb_init.py:init():834] backend started and connected
+2026-07-17 15:43:32,384 INFO    MainThread:499773 [wandb_init.py:init():904] updated telemetry
+2026-07-17 15:43:32,392 INFO    MainThread:499773 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-17 15:43:32,831 INFO    MainThread:499773 [wandb_init.py:init():972] starting run threads in backend
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-17 15:43:33,016 INFO    MainThread:499773 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/RSCLstyle_cosine_60K.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-e5avxg6w/files/output.log

@@ -0,0 +1,217 @@
+INFO 2026-07-17 15:43:33 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-17 15:43:33 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/e5avxg6w
+INFO 2026-07-17 15:43:33 ot_train.py:298 Creating dataset
+INFO 2026-07-17 15:43:36 ot_train.py:332 Creating policy
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 1539.30it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1211.10it/s]
+INFO 2026-07-17 15:43:43 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-17 15:43:43 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K
+INFO 2026-07-17 15:43:43 ot_train.py:446 cfg.steps=60000 (60K)
+INFO 2026-07-17 15:43:43 ot_train.py:447 dataset.num_frames=218367 (218K)
+INFO 2026-07-17 15:43:43 ot_train.py:448 dataset.num_episodes=4930
+INFO 2026-07-17 15:43:43 ot_train.py:451 Effective batch size: 128 x 1 = 128
+INFO 2026-07-17 15:43:43 ot_train.py:452 num_learnable_params=1622090881 (2B)
+INFO 2026-07-17 15:43:43 ot_train.py:453 num_total_params=3145712033 (3B)
+Training:   0%|          | 0/60000 [00:00<?, ?step/s]INFO 2026-07-17 15:43:43 ot_train.py:604 Start offline training on a fixed dataset, with effective batch size: 128
+Training:   0%|          | 200/60000 [17:39<58:00:38,  3.49s/step]INFO 2026-07-17 16:01:22 ot_train.py:649 step:200 smpl:26K ep:578 epch:0.12 loss:1.201 grdn:0.511 lr:3.3e-06 updt_s:1.738 data_s:3.553 smp/s:24 mem_gb:52.89
+Training:   1%|          | 400/60000 [28:42<59:39:42,  3.60s/step]INFO 2026-07-17 16:12:25 ot_train.py:649 step:400 smpl:51K ep:1K epch:0.23 loss:0.551 grdn:1.400 lr:1.0e-05 updt_s:1.642 data_s:1.669 smp/s:39 mem_gb:52.95
+Training:   1%|          | 600/60000 [39:43<54:20:16,  3.29s/step]INFO 2026-07-17 16:23:27 ot_train.py:649 step:600 smpl:77K ep:2K epch:0.35 loss:0.225 grdn:2.025 lr:1.7e-05 updt_s:1.667 data_s:1.634 smp/s:39 mem_gb:52.95
+Training:   1%|▏         | 800/60000 [50:22<54:11:17,  3.30s/step]INFO 2026-07-17 16:34:05 ot_train.py:649 step:800 smpl:102K ep:2K epch:0.47 loss:0.174 grdn:2.038 lr:2.3e-05 updt_s:1.666 data_s:1.521 smp/s:40 mem_gb:52.95
+Training:   2%|▏         | 1000/60000 [1:01:47<55:06:23,  3.36s/step]INFO 2026-07-17 16:45:30 ot_train.py:649 step:1K smpl:128K ep:3K epch:0.59 loss:0.149 grdn:1.800 lr:3.0e-05 updt_s:1.736 data_s:1.683 smp/s:37 mem_gb:52.95
+Training:   2%|▏         | 1200/60000 [1:13:31<59:35:47,  3.65s/step]INFO 2026-07-17 16:57:14 ot_train.py:649 step:1K smpl:154K ep:3K epch:0.70 loss:0.136 grdn:1.597 lr:3.7e-05 updt_s:1.748 data_s:1.764 smp/s:36 mem_gb:52.88
+Training:   2%|▏         | 1400/60000 [1:25:09<62:20:15,  3.83s/step]INFO 2026-07-17 17:08:52 ot_train.py:649 step:1K smpl:179K ep:4K epch:0.82 loss:0.127 grdn:1.406 lr:4.3e-05 updt_s:1.760 data_s:1.725 smp/s:37 mem_gb:52.98
+Training:   3%|▎         | 1600/60000 [1:36:48<52:39:12,  3.25s/step]INFO 2026-07-17 17:20:31 ot_train.py:649 step:2K smpl:205K ep:5K epch:0.94 loss:0.120 grdn:1.185 lr:5.0e-05 updt_s:1.737 data_s:1.751 smp/s:37 mem_gb:52.98
+Training:   3%|▎         | 1800/60000 [1:47:30<52:34:15,  3.25s/step]INFO 2026-07-17 17:31:14 ot_train.py:649 step:2K smpl:230K ep:5K epch:1.06 loss:0.114 grdn:1.003 lr:5.7e-05 updt_s:1.593 data_s:1.614 smp/s:40 mem_gb:52.98
+Training:   3%|▎         | 2000/60000 [1:58:01<50:56:36,  3.16s/step]INFO 2026-07-17 17:41:45 ot_train.py:649 step:2K smpl:256K ep:6K epch:1.17 loss:0.106 grdn:0.866 lr:6.3e-05 updt_s:1.565 data_s:1.585 smp/s:41 mem_gb:52.98
+Training:   4%|▎         | 2200/60000 [2:08:30<50:24:36,  3.14s/step]INFO 2026-07-17 17:52:13 ot_train.py:649 step:2K smpl:282K ep:6K epch:1.29 loss:0.099 grdn:0.742 lr:7.0e-05 updt_s:1.525 data_s:1.610 smp/s:41 mem_gb:52.98
+Training:   4%|▍         | 2400/60000 [2:18:57<49:42:18,  3.11s/step]INFO 2026-07-17 18:02:41 ot_train.py:649 step:2K smpl:307K ep:7K epch:1.41 loss:0.095 grdn:0.686 lr:7.7e-05 updt_s:1.528 data_s:1.604 smp/s:41 mem_gb:52.90
+Training:   4%|▍         | 2600/60000 [2:30:03<57:21:34,  3.60s/step]INFO 2026-07-17 18:13:46 ot_train.py:649 step:3K smpl:333K ep:8K epch:1.52 loss:0.091 grdn:0.634 lr:8.3e-05 updt_s:1.638 data_s:1.683 smp/s:39 mem_gb:52.98
+Training:   5%|▍         | 2800/60000 [2:41:33<51:22:00,  3.23s/step]INFO 2026-07-17 18:25:16 ot_train.py:649 step:3K smpl:358K ep:8K epch:1.64 loss:0.088 grdn:0.591 lr:9.0e-05 updt_s:1.757 data_s:1.687 smp/s:37 mem_gb:52.98
+Training:   5%|▌         | 3000/60000 [2:52:59<56:22:12,  3.56s/step]INFO 2026-07-17 18:36:43 ot_train.py:649 step:3K smpl:384K ep:9K epch:1.76 loss:0.085 grdn:0.561 lr:9.7e-05 updt_s:1.676 data_s:1.750 smp/s:37 mem_gb:52.98
+Training:   5%|▌         | 3200/60000 [3:04:43<58:39:53,  3.72s/step]INFO 2026-07-17 18:48:27 ot_train.py:649 step:3K smpl:410K ep:9K epch:1.88 loss:0.082 grdn:0.502 lr:1.0e-04 updt_s:1.739 data_s:1.775 smp/s:36 mem_gb:52.98
+Training:   6%|▌         | 3400/60000 [3:16:14<56:32:11,  3.60s/step]INFO 2026-07-17 18:59:57 ot_train.py:649 step:3K smpl:435K ep:10K epch:1.99 loss:0.080 grdn:0.472 lr:1.0e-04 updt_s:1.731 data_s:1.715 smp/s:37 mem_gb:52.90
+Training:   6%|▌         | 3600/60000 [3:27:25<52:30:05,  3.35s/step]INFO 2026-07-17 19:11:09 ot_train.py:649 step:4K smpl:461K ep:10K epch:2.11 loss:0.075 grdn:0.447 lr:1.0e-04 updt_s:1.662 data_s:1.691 smp/s:38 mem_gb:52.98
+Training:   6%|▋         | 3800/60000 [3:39:13<57:48:44,  3.70s/step]INFO 2026-07-17 19:22:56 ot_train.py:649 step:4K smpl:486K ep:11K epch:2.23 loss:0.074 grdn:0.431 lr:1.0e-04 updt_s:1.697 data_s:1.836 smp/s:36 mem_gb:52.98
+Training:   7%|▋         | 4000/60000 [3:50:47<55:20:45,  3.56s/step]INFO 2026-07-17 19:34:30 ot_train.py:649 step:4K smpl:512K ep:12K epch:2.34 loss:0.072 grdn:0.411 lr:1.0e-04 updt_s:1.685 data_s:1.777 smp/s:37 mem_gb:52.98
+Training:   7%|▋         | 4200/60000 [4:03:36<55:04:16,  3.55s/step]INFO 2026-07-17 19:47:20 ot_train.py:649 step:4K smpl:538K ep:12K epch:2.46 loss:0.070 grdn:0.383 lr:1.0e-04 updt_s:1.742 data_s:2.100 smp/s:33 mem_gb:52.98
+Training:   7%|▋         | 4400/60000 [4:15:47<54:17:17,  3.52s/step]INFO 2026-07-17 19:59:31 ot_train.py:649 step:4K smpl:563K ep:13K epch:2.58 loss:0.069 grdn:0.382 lr:1.0e-04 updt_s:1.775 data_s:1.873 smp/s:35 mem_gb:52.98
+Training:   8%|▊         | 4600/60000 [4:28:09<62:31:58,  4.06s/step]INFO 2026-07-17 20:11:52 ot_train.py:649 step:5K smpl:589K ep:13K epch:2.70 loss:0.068 grdn:0.374 lr:1.0e-04 updt_s:1.726 data_s:1.975 smp/s:35 mem_gb:52.90
+Training:   8%|▊         | 4800/60000 [4:40:28<45:52:40,  2.99s/step]INFO 2026-07-17 20:24:11 ot_train.py:649 step:5K smpl:614K ep:14K epch:2.81 loss:0.066 grdn:0.365 lr:1.0e-04 updt_s:1.817 data_s:1.874 smp/s:35 mem_gb:52.98
+Training:   8%|▊         | 5000/60000 [4:53:04<57:27:14,  3.76s/step]INFO 2026-07-17 20:36:48 ot_train.py:649 step:5K smpl:640K ep:14K epch:2.93 loss:0.065 grdn:0.361 lr:1.0e-04 updt_s:1.823 data_s:1.953 smp/s:34 mem_gb:52.98
+Training:   9%|▊         | 5200/60000 [5:05:25<54:52:02,  3.60s/step]INFO 2026-07-17 20:49:09 ot_train.py:649 step:5K smpl:666K ep:15K epch:3.05 loss:0.065 grdn:0.358 lr:1.0e-04 updt_s:1.769 data_s:1.930 smp/s:35 mem_gb:52.98
+Training:   9%|▉         | 5400/60000 [5:17:03<56:24:04,  3.72s/step]INFO 2026-07-17 21:00:46 ot_train.py:649 step:5K smpl:691K ep:16K epch:3.17 loss:0.062 grdn:0.330 lr:1.0e-04 updt_s:1.745 data_s:1.734 smp/s:37 mem_gb:52.98
+Training:   9%|▉         | 5600/60000 [5:27:52<50:14:46,  3.33s/step]INFO 2026-07-17 21:11:36 ot_train.py:649 step:6K smpl:717K ep:16K epch:3.28 loss:0.061 grdn:0.324 lr:1.0e-04 updt_s:1.674 data_s:1.570 smp/s:39 mem_gb:52.98
+Training:  10%|▉         | 5800/60000 [5:39:06<49:16:06,  3.27s/step]INFO 2026-07-17 21:22:50 ot_train.py:649 step:6K smpl:742K ep:17K epch:3.40 loss:0.059 grdn:0.336 lr:9.9e-05 updt_s:1.727 data_s:1.636 smp/s:38 mem_gb:52.90
+Training:  10%|█         | 6000/60000 [5:49:58<46:20:45,  3.09s/step]INFO 2026-07-17 21:33:42 ot_train.py:649 step:6K smpl:768K ep:17K epch:3.52 loss:0.060 grdn:0.323 lr:9.9e-05 updt_s:1.690 data_s:1.565 smp/s:39 mem_gb:52.98
+Training:  10%|█         | 6200/60000 [6:00:45<62:35:59,  4.19s/step]INFO 2026-07-17 21:44:29 ot_train.py:649 step:6K smpl:794K ep:18K epch:3.63 loss:0.059 grdn:0.318 lr:9.9e-05 updt_s:1.623 data_s:1.607 smp/s:40 mem_gb:52.98
+Training:  11%|█         | 6400/60000 [6:11:20<48:48:46,  3.28s/step]INFO 2026-07-17 21:55:03 ot_train.py:649 step:6K smpl:819K ep:18K epch:3.75 loss:0.058 grdn:0.325 lr:9.9e-05 updt_s:1.587 data_s:1.578 smp/s:40 mem_gb:52.98
+Training:  11%|█         | 6600/60000 [6:21:47<45:05:55,  3.04s/step]INFO 2026-07-17 22:05:31 ot_train.py:649 step:7K smpl:845K ep:19K epch:3.87 loss:0.059 grdn:0.321 lr:9.9e-05 updt_s:1.560 data_s:1.572 smp/s:41 mem_gb:52.98
+Training:  11%|█▏        | 6800/60000 [6:32:13<43:48:46,  2.96s/step]INFO 2026-07-17 22:15:56 ot_train.py:649 step:7K smpl:870K ep:20K epch:3.99 loss:0.056 grdn:0.301 lr:9.9e-05 updt_s:1.593 data_s:1.531 smp/s:41 mem_gb:52.90
+Training:  12%|█▏        | 7000/60000 [6:42:34<45:19:47,  3.08s/step]INFO 2026-07-17 22:26:17 ot_train.py:649 step:7K smpl:896K ep:20K epch:4.10 loss:0.054 grdn:0.301 lr:9.9e-05 updt_s:1.650 data_s:1.449 smp/s:41 mem_gb:52.98
+Training:  12%|█▏        | 7200/60000 [6:52:35<43:35:06,  2.97s/step]INFO 2026-07-17 22:36:19 ot_train.py:649 step:7K smpl:922K ep:21K epch:4.22 loss:0.054 grdn:0.299 lr:9.9e-05 updt_s:1.627 data_s:1.374 smp/s:43 mem_gb:52.98
+Training:  12%|█▏        | 7400/60000 [7:03:08<44:56:08,  3.08s/step]INFO 2026-07-17 22:46:51 ot_train.py:649 step:7K smpl:947K ep:21K epch:4.34 loss:0.054 grdn:0.307 lr:9.9e-05 updt_s:1.671 data_s:1.485 smp/s:41 mem_gb:52.98
+Training:  13%|█▎        | 7600/60000 [7:13:44<45:25:05,  3.12s/step]INFO 2026-07-17 22:57:27 ot_train.py:649 step:8K smpl:973K ep:22K epch:4.45 loss:0.054 grdn:0.292 lr:9.8e-05 updt_s:1.662 data_s:1.514 smp/s:40 mem_gb:52.98
+Training:  13%|█▎        | 7800/60000 [7:24:19<44:48:00,  3.09s/step]INFO 2026-07-17 23:08:02 ot_train.py:649 step:8K smpl:998K ep:23K epch:4.57 loss:0.053 grdn:0.289 lr:9.8e-05 updt_s:1.671 data_s:1.499 smp/s:40 mem_gb:52.98
+Training:  13%|█▎        | 8000/60000 [7:34:43<45:41:16,  3.16s/step]INFO 2026-07-17 23:18:27 ot_train.py:649 step:8K smpl:1M ep:23K epch:4.69 loss:0.052 grdn:0.297 lr:9.8e-05 updt_s:1.642 data_s:1.474 smp/s:41 mem_gb:52.90
+Training:  14%|█▎        | 8200/60000 [7:45:00<44:19:13,  3.08s/step]INFO 2026-07-17 23:28:43 ot_train.py:649 step:8K smpl:1M ep:24K epch:4.81 loss:0.051 grdn:0.295 lr:9.8e-05 updt_s:1.602 data_s:1.474 smp/s:42 mem_gb:52.98
+Training:  14%|█▍        | 8400/60000 [7:55:20<44:20:26,  3.09s/step]INFO 2026-07-17 23:39:04 ot_train.py:649 step:8K smpl:1M ep:24K epch:4.92 loss:0.050 grdn:0.289 lr:9.8e-05 updt_s:1.603 data_s:1.495 smp/s:41 mem_gb:52.98
+Training:  14%|█▍        | 8600/60000 [8:05:50<42:03:33,  2.95s/step]INFO 2026-07-17 23:49:34 ot_train.py:649 step:9K smpl:1M ep:25K epch:5.04 loss:0.050 grdn:0.281 lr:9.8e-05 updt_s:1.600 data_s:1.545 smp/s:41 mem_gb:52.98
+Training:  15%|█▍        | 8800/60000 [8:16:03<42:50:30,  3.01s/step]INFO 2026-07-17 23:59:46 ot_train.py:649 step:9K smpl:1M ep:25K epch:5.16 loss:0.049 grdn:0.283 lr:9.8e-05 updt_s:1.533 data_s:1.523 smp/s:42 mem_gb:52.98
+Training:  15%|█▌        | 9000/60000 [8:26:35<47:00:40,  3.32s/step]INFO 2026-07-18 00:10:18 ot_train.py:649 step:9K smpl:1M ep:26K epch:5.28 loss:0.050 grdn:0.272 lr:9.7e-05 updt_s:1.621 data_s:1.535 smp/s:41 mem_gb:52.98
+Training:  15%|█▌        | 9200/60000 [8:37:11<42:46:43,  3.03s/step]INFO 2026-07-18 00:20:55 ot_train.py:649 step:9K smpl:1M ep:27K epch:5.39 loss:0.047 grdn:0.274 lr:9.7e-05 updt_s:1.669 data_s:1.508 smp/s:40 mem_gb:52.90
+Training:  16%|█▌        | 9400/60000 [8:47:52<46:46:03,  3.33s/step]INFO 2026-07-18 00:31:36 ot_train.py:649 step:9K smpl:1M ep:27K epch:5.51 loss:0.048 grdn:0.287 lr:9.7e-05 updt_s:1.701 data_s:1.498 smp/s:40 mem_gb:52.98
+Training:  16%|█▌        | 9600/60000 [8:58:14<41:41:27,  2.98s/step]INFO 2026-07-18 00:41:57 ot_train.py:649 step:10K smpl:1M ep:28K epch:5.63 loss:0.049 grdn:0.289 lr:9.7e-05 updt_s:1.649 data_s:1.454 smp/s:41 mem_gb:52.98
+Training:  16%|█▋        | 9800/60000 [9:08:20<37:20:36,  2.68s/step]INFO 2026-07-18 00:52:03 ot_train.py:649 step:10K smpl:1M ep:28K epch:5.74 loss:0.047 grdn:0.283 lr:9.7e-05 updt_s:1.602 data_s:1.422 smp/s:42 mem_gb:52.98
+Training:  17%|█▋        | 10000/60000 [9:18:17<39:48:10,  2.87s/step]INFO 2026-07-18 01:02:00 ot_train.py:649 step:10K smpl:1M ep:29K epch:5.86 loss:0.047 grdn:0.270 lr:9.6e-05 updt_s:1.630 data_s:1.350 smp/s:43 mem_gb:52.98
+Training:  17%|█▋        | 10200/60000 [9:28:17<39:01:40,  2.82s/step]INFO 2026-07-18 01:12:01 ot_train.py:649 step:10K smpl:1M ep:29K epch:5.98 loss:0.046 grdn:0.268 lr:9.6e-05 updt_s:1.621 data_s:1.377 smp/s:43 mem_gb:52.90
+Training:  17%|█▋        | 10400/60000 [9:38:04<42:25:27,  3.08s/step]INFO 2026-07-18 01:21:47 ot_train.py:649 step:10K smpl:1M ep:30K epch:6.10 loss:0.044 grdn:0.265 lr:9.6e-05 updt_s:1.577 data_s:1.350 smp/s:44 mem_gb:52.98
+Training:  18%|█▊        | 10600/60000 [9:47:54<39:43:29,  2.89s/step]INFO 2026-07-18 01:31:38 ot_train.py:649 step:11K smpl:1M ep:31K epch:6.21 loss:0.044 grdn:0.265 lr:9.6e-05 updt_s:1.588 data_s:1.362 smp/s:43 mem_gb:52.98
+Training:  18%|█▊        | 10800/60000 [9:58:10<41:34:29,  3.04s/step]INFO 2026-07-18 01:41:54 ot_train.py:649 step:11K smpl:1M ep:31K epch:6.33 loss:0.046 grdn:0.276 lr:9.6e-05 updt_s:1.574 data_s:1.499 smp/s:42 mem_gb:52.98
+Training:  18%|█▊        | 11000/60000 [10:08:16<42:46:18,  3.14s/step]INFO 2026-07-18 01:51:59 ot_train.py:649 step:11K smpl:1M ep:32K epch:6.45 loss:0.044 grdn:0.262 lr:9.5e-05 updt_s:1.570 data_s:1.454 smp/s:42 mem_gb:52.98
+Training:  19%|█▊        | 11200/60000 [10:18:16<42:38:38,  3.15s/step]INFO 2026-07-18 02:01:59 ot_train.py:649 step:11K smpl:1M ep:32K epch:6.57 loss:0.044 grdn:0.269 lr:9.5e-05 updt_s:1.571 data_s:1.421 smp/s:43 mem_gb:52.98
+Training:  19%|█▉        | 11400/60000 [10:28:16<41:31:54,  3.08s/step]INFO 2026-07-18 02:12:00 ot_train.py:649 step:11K smpl:1M ep:33K epch:6.68 loss:0.041 grdn:0.250 lr:9.5e-05 updt_s:1.578 data_s:1.419 smp/s:43 mem_gb:52.90
+Training:  19%|█▉        | 11600/60000 [10:38:39<42:21:24,  3.15s/step]INFO 2026-07-18 02:22:22 ot_train.py:649 step:12K smpl:1M ep:34K epch:6.80 loss:0.040 grdn:0.253 lr:9.5e-05 updt_s:1.584 data_s:1.524 smp/s:41 mem_gb:52.98
+Training:  20%|█▉        | 11800/60000 [10:49:31<42:08:36,  3.15s/step]INFO 2026-07-18 02:33:14 ot_train.py:649 step:12K smpl:2M ep:34K epch:6.92 loss:0.041 grdn:0.260 lr:9.4e-05 updt_s:1.673 data_s:1.583 smp/s:39 mem_gb:52.98
+Training:  20%|██        | 12000/60000 [10:59:53<39:31:53,  2.96s/step]INFO 2026-07-18 02:43:36 ot_train.py:649 step:12K smpl:2M ep:35K epch:7.03 loss:0.043 grdn:0.274 lr:9.4e-05 updt_s:1.573 data_s:1.531 smp/s:41 mem_gb:52.98
+Training:  20%|██        | 12200/60000 [11:09:36<33:52:37,  2.55s/step]INFO 2026-07-18 02:53:20 ot_train.py:649 step:12K smpl:2M ep:35K epch:7.15 loss:0.042 grdn:0.265 lr:9.4e-05 updt_s:1.593 data_s:1.319 smp/s:44 mem_gb:52.98
+Training:  21%|██        | 12400/60000 [11:18:57<40:53:08,  3.09s/step]INFO 2026-07-18 03:02:41 ot_train.py:649 step:12K smpl:2M ep:36K epch:7.27 loss:0.041 grdn:0.262 lr:9.4e-05 updt_s:1.569 data_s:1.232 smp/s:46 mem_gb:52.98
+Training:  21%|██        | 12600/60000 [11:28:21<38:07:28,  2.90s/step]INFO 2026-07-18 03:12:05 ot_train.py:649 step:13K smpl:2M ep:36K epch:7.39 loss:0.038 grdn:0.251 lr:9.3e-05 updt_s:1.625 data_s:1.191 smp/s:45 mem_gb:52.90
+Training:  21%|██▏       | 12800/60000 [11:37:43<37:43:54,  2.88s/step]INFO 2026-07-18 03:21:26 ot_train.py:649 step:13K smpl:2M ep:37K epch:7.50 loss:0.040 grdn:0.257 lr:9.3e-05 updt_s:1.588 data_s:1.214 smp/s:46 mem_gb:52.98
+Training:  22%|██▏       | 13000/60000 [11:47:10<39:56:40,  3.06s/step]INFO 2026-07-18 03:30:53 ot_train.py:649 step:13K smpl:2M ep:38K epch:7.62 loss:0.039 grdn:0.253 lr:9.3e-05 updt_s:1.585 data_s:1.247 smp/s:45 mem_gb:52.98
+Training:  22%|██▏       | 13200/60000 [11:56:34<32:54:43,  2.53s/step]INFO 2026-07-18 03:40:17 ot_train.py:649 step:13K smpl:2M ep:38K epch:7.74 loss:0.038 grdn:0.253 lr:9.2e-05 updt_s:1.595 data_s:1.221 smp/s:45 mem_gb:52.98
+Training:  22%|██▏       | 13400/60000 [12:05:52<37:04:16,  2.86s/step]INFO 2026-07-18 03:49:35 ot_train.py:649 step:13K smpl:2M ep:39K epch:7.85 loss:0.038 grdn:0.250 lr:9.2e-05 updt_s:1.593 data_s:1.190 smp/s:46 mem_gb:52.98
+Training:  23%|██▎       | 13600/60000 [12:15:18<37:31:11,  2.91s/step]INFO 2026-07-18 03:59:02 ot_train.py:649 step:14K smpl:2M ep:39K epch:7.97 loss:0.037 grdn:0.237 lr:9.2e-05 updt_s:1.607 data_s:1.221 smp/s:45 mem_gb:52.90
+Training:  23%|██▎       | 13800/60000 [12:24:44<36:49:16,  2.87s/step]INFO 2026-07-18 04:08:27 ot_train.py:649 step:14K smpl:2M ep:40K epch:8.09 loss:0.034 grdn:0.236 lr:9.2e-05 updt_s:1.600 data_s:1.223 smp/s:45 mem_gb:52.98
+Training:  23%|██▎       | 14000/60000 [12:34:18<38:03:33,  2.98s/step]INFO 2026-07-18 04:18:01 ot_train.py:649 step:14K smpl:2M ep:40K epch:8.21 loss:0.036 grdn:0.251 lr:9.1e-05 updt_s:1.604 data_s:1.261 smp/s:45 mem_gb:52.98
+Training:  24%|██▎       | 14200/60000 [12:44:03<35:23:01,  2.78s/step]INFO 2026-07-18 04:27:47 ot_train.py:649 step:14K smpl:2M ep:41K epch:8.32 loss:0.037 grdn:0.248 lr:9.1e-05 updt_s:1.639 data_s:1.284 smp/s:44 mem_gb:52.98
+Training:  24%|██▍       | 14400/60000 [12:53:43<36:48:06,  2.91s/step]INFO 2026-07-18 04:37:27 ot_train.py:649 step:14K smpl:2M ep:42K epch:8.44 loss:0.042 grdn:0.281 lr:9.1e-05 updt_s:1.636 data_s:1.260 smp/s:44 mem_gb:52.98
+Training:  24%|██▍       | 14600/60000 [13:03:25<37:51:57,  3.00s/step]INFO 2026-07-18 04:47:08 ot_train.py:649 step:15K smpl:2M ep:42K epch:8.56 loss:0.037 grdn:0.246 lr:9.0e-05 updt_s:1.613 data_s:1.290 smp/s:44 mem_gb:52.98
+Training:  25%|██▍       | 14800/60000 [13:12:52<34:29:50,  2.75s/step]INFO 2026-07-18 04:56:36 ot_train.py:649 step:15K smpl:2M ep:43K epch:8.68 loss:0.035 grdn:0.238 lr:9.0e-05 updt_s:1.577 data_s:1.257 smp/s:45 mem_gb:52.90
+Training:  25%|██▌       | 15000/60000 [13:22:17<33:35:38,  2.69s/step]INFO 2026-07-18 05:06:01 ot_train.py:649 step:15K smpl:2M ep:43K epch:8.79 loss:0.035 grdn:0.243 lr:9.0e-05 updt_s:1.591 data_s:1.230 smp/s:45 mem_gb:52.98
+Training:  25%|██▌       | 15200/60000 [13:31:57<35:23:33,  2.84s/step]INFO 2026-07-18 05:15:40 ot_train.py:649 step:15K smpl:2M ep:44K epch:8.91 loss:0.034 grdn:0.237 lr:8.9e-05 updt_s:1.609 data_s:1.283 smp/s:44 mem_gb:52.98
+Training:  26%|██▌       | 15400/60000 [13:41:46<35:51:10,  2.89s/step]INFO 2026-07-18 05:25:30 ot_train.py:649 step:15K smpl:2M ep:45K epch:9.03 loss:0.035 grdn:0.241 lr:8.9e-05 updt_s:1.617 data_s:1.326 smp/s:43 mem_gb:52.98
+Training:  26%|██▌       | 15600/60000 [13:51:35<34:40:12,  2.81s/step]INFO 2026-07-18 05:35:18 ot_train.py:649 step:16K smpl:2M ep:45K epch:9.14 loss:0.034 grdn:0.235 lr:8.9e-05 updt_s:1.612 data_s:1.326 smp/s:44 mem_gb:52.98
+Training:  26%|██▋       | 15800/60000 [14:01:31<34:09:24,  2.78s/step]INFO 2026-07-18 05:45:14 ot_train.py:649 step:16K smpl:2M ep:46K epch:9.26 loss:0.034 grdn:0.227 lr:8.8e-05 updt_s:1.627 data_s:1.347 smp/s:43 mem_gb:52.98
+Training:  27%|██▋       | 16000/60000 [14:11:29<36:18:57,  2.97s/step]INFO 2026-07-18 05:55:12 ot_train.py:649 step:16K smpl:2M ep:46K epch:9.38 loss:0.032 grdn:0.229 lr:8.8e-05 updt_s:1.618 data_s:1.369 smp/s:43 mem_gb:52.90
+Training:  27%|██▋       | 16200/60000 [14:21:33<35:34:14,  2.92s/step]INFO 2026-07-18 06:05:16 ot_train.py:649 step:16K smpl:2M ep:47K epch:9.50 loss:0.033 grdn:0.241 lr:8.8e-05 updt_s:1.607 data_s:1.407 smp/s:42 mem_gb:52.98
+Training:  27%|██▋       | 16400/60000 [14:31:45<32:46:00,  2.71s/step]INFO 2026-07-18 06:15:28 ot_train.py:649 step:16K smpl:2M ep:47K epch:9.61 loss:0.032 grdn:0.233 lr:8.7e-05 updt_s:1.667 data_s:1.389 smp/s:42 mem_gb:52.98
+Training:  28%|██▊       | 16600/60000 [14:41:44<33:40:20,  2.79s/step]INFO 2026-07-18 06:25:27 ot_train.py:649 step:17K smpl:2M ep:48K epch:9.73 loss:0.032 grdn:0.225 lr:8.7e-05 updt_s:1.649 data_s:1.342 smp/s:43 mem_gb:52.98
+Training:  28%|██▊       | 16800/60000 [14:51:44<35:16:39,  2.94s/step]INFO 2026-07-18 06:35:28 ot_train.py:649 step:17K smpl:2M ep:49K epch:9.85 loss:0.032 grdn:0.225 lr:8.6e-05 updt_s:1.624 data_s:1.374 smp/s:43 mem_gb:52.98
+Training:  28%|██▊       | 17000/60000 [15:01:40<36:19:56,  3.04s/step]INFO 2026-07-18 06:45:23 ot_train.py:649 step:17K smpl:2M ep:49K epch:9.96 loss:0.031 grdn:0.215 lr:8.6e-05 updt_s:1.629 data_s:1.344 smp/s:43 mem_gb:52.90
+Training:  29%|██▊       | 17200/60000 [15:11:24<33:25:31,  2.81s/step]INFO 2026-07-18 06:55:08 ot_train.py:649 step:17K smpl:2M ep:50K epch:10.08 loss:0.030 grdn:0.220 lr:8.6e-05 updt_s:1.648 data_s:1.270 smp/s:44 mem_gb:52.98
+Training:  29%|██▉       | 17400/60000 [15:20:56<33:00:29,  2.79s/step]INFO 2026-07-18 07:04:39 ot_train.py:649 step:17K smpl:2M ep:50K epch:10.20 loss:0.030 grdn:0.225 lr:8.5e-05 updt_s:1.596 data_s:1.256 smp/s:45 mem_gb:52.98
+Training:  29%|██▉       | 17600/60000 [15:30:18<33:50:42,  2.87s/step]INFO 2026-07-18 07:14:02 ot_train.py:649 step:18K smpl:2M ep:51K epch:10.32 loss:0.031 grdn:0.220 lr:8.5e-05 updt_s:1.623 data_s:1.187 smp/s:46 mem_gb:52.98
+Training:  30%|██▉       | 17800/60000 [15:39:44<34:15:03,  2.92s/step]INFO 2026-07-18 07:23:27 ot_train.py:649 step:18K smpl:2M ep:51K epch:10.43 loss:0.032 grdn:0.235 lr:8.4e-05 updt_s:1.602 data_s:1.221 smp/s:45 mem_gb:52.98
+Training:  30%|███       | 18000/60000 [15:48:55<31:20:10,  2.69s/step]INFO 2026-07-18 07:32:39 ot_train.py:649 step:18K smpl:2M ep:52K epch:10.55 loss:0.031 grdn:0.229 lr:8.4e-05 updt_s:1.581 data_s:1.173 smp/s:46 mem_gb:52.98
+Training:  30%|███       | 18200/60000 [15:58:17<31:23:35,  2.70s/step]INFO 2026-07-18 07:42:00 ot_train.py:649 step:18K smpl:2M ep:53K epch:10.67 loss:0.030 grdn:0.216 lr:8.4e-05 updt_s:1.555 data_s:1.249 smp/s:46 mem_gb:52.90
+Training:  31%|███       | 18400/60000 [16:07:44<33:14:07,  2.88s/step]INFO 2026-07-18 07:51:27 ot_train.py:649 step:18K smpl:2M ep:53K epch:10.79 loss:0.028 grdn:0.219 lr:8.3e-05 updt_s:1.600 data_s:1.230 smp/s:45 mem_gb:52.98
+Training:  31%|███       | 18600/60000 [16:16:55<32:16:04,  2.81s/step]INFO 2026-07-18 08:00:39 ot_train.py:649 step:19K smpl:2M ep:54K epch:10.90 loss:0.029 grdn:0.215 lr:8.3e-05 updt_s:1.557 data_s:1.197 smp/s:46 mem_gb:52.98
+Training:  31%|███▏      | 18800/60000 [16:26:11<32:25:43,  2.83s/step]INFO 2026-07-18 08:09:54 ot_train.py:649 step:19K smpl:2M ep:54K epch:11.02 loss:0.029 grdn:0.224 lr:8.2e-05 updt_s:1.553 data_s:1.217 smp/s:46 mem_gb:52.98
+Training:  32%|███▏      | 19000/60000 [16:35:51<31:10:54,  2.74s/step]INFO 2026-07-18 08:19:35 ot_train.py:649 step:19K smpl:2M ep:55K epch:11.14 loss:0.028 grdn:0.216 lr:8.2e-05 updt_s:1.617 data_s:1.283 smp/s:44 mem_gb:52.98
+Training:  32%|███▏      | 19200/60000 [16:45:40<33:54:02,  2.99s/step]INFO 2026-07-18 08:29:24 ot_train.py:649 step:19K smpl:2M ep:55K epch:11.25 loss:0.030 grdn:0.219 lr:8.2e-05 updt_s:1.602 data_s:1.339 smp/s:44 mem_gb:52.90
+Training:  32%|███▏      | 19400/60000 [16:55:14<33:21:00,  2.96s/step]INFO 2026-07-18 08:38:57 ot_train.py:649 step:19K smpl:2M ep:56K epch:11.37 loss:0.027 grdn:0.209 lr:8.1e-05 updt_s:1.621 data_s:1.243 smp/s:45 mem_gb:52.98
+Training:  33%|███▎      | 19600/60000 [17:04:42<31:17:46,  2.79s/step]INFO 2026-07-18 08:48:25 ot_train.py:649 step:20K smpl:3M ep:57K epch:11.49 loss:0.027 grdn:0.212 lr:8.1e-05 updt_s:1.598 data_s:1.237 smp/s:45 mem_gb:52.98
+Training:  33%|███▎      | 19800/60000 [17:14:19<34:00:47,  3.05s/step]INFO 2026-07-18 08:58:02 ot_train.py:649 step:20K smpl:3M ep:57K epch:11.61 loss:0.027 grdn:0.209 lr:8.0e-05 updt_s:1.621 data_s:1.261 smp/s:44 mem_gb:52.98
+Training:  33%|███▎      | 20000/60000 [17:24:09<30:33:19,  2.75s/step]INFO 2026-07-18 09:07:52 ot_train.py:649 step:20K smpl:3M ep:58K epch:11.72 loss:0.028 grdn:0.219 lr:8.0e-05 updt_s:1.635 data_s:1.308 smp/s:43 mem_gb:52.98
+Training:  34%|███▎      | 20200/60000 [17:33:58<30:58:48,  2.80s/step]INFO 2026-07-18 09:17:41 ot_train.py:649 step:20K smpl:3M ep:58K epch:11.84 loss:0.027 grdn:0.209 lr:7.9e-05 updt_s:1.641 data_s:1.301 smp/s:44 mem_gb:52.98
+Training:  34%|███▍      | 20400/60000 [17:43:53<37:40:07,  3.42s/step]INFO 2026-07-18 09:27:36 ot_train.py:649 step:20K smpl:3M ep:59K epch:11.96 loss:0.026 grdn:0.207 lr:7.9e-05 updt_s:1.606 data_s:1.365 smp/s:43 mem_gb:52.90
+Training:  34%|███▍      | 20600/60000 [17:54:06<38:30:53,  3.52s/step]INFO 2026-07-18 09:37:49 ot_train.py:649 step:21K smpl:3M ep:60K epch:12.08 loss:0.025 grdn:0.192 lr:7.8e-05 updt_s:1.679 data_s:1.380 smp/s:42 mem_gb:52.98
+Training:  35%|███▍      | 20800/60000 [18:05:12<35:27:33,  3.26s/step]INFO 2026-07-18 09:48:55 ot_train.py:649 step:21K smpl:3M ep:60K epch:12.19 loss:0.026 grdn:0.214 lr:7.8e-05 updt_s:1.707 data_s:1.617 smp/s:39 mem_gb:52.98
+Training:  35%|███▌      | 21000/60000 [18:16:28<34:59:30,  3.23s/step]INFO 2026-07-18 10:00:11 ot_train.py:649 step:21K smpl:3M ep:61K epch:12.31 loss:0.026 grdn:0.205 lr:7.8e-05 updt_s:1.718 data_s:1.656 smp/s:38 mem_gb:52.98
+Training:  35%|███▌      | 21200/60000 [18:27:23<38:59:11,  3.62s/step]INFO 2026-07-18 10:11:07 ot_train.py:649 step:21K smpl:3M ep:61K epch:12.43 loss:0.026 grdn:0.210 lr:7.7e-05 updt_s:1.696 data_s:1.576 smp/s:39 mem_gb:52.98
+Training:  36%|███▌      | 21400/60000 [18:38:38<36:58:59,  3.45s/step]INFO 2026-07-18 10:22:22 ot_train.py:649 step:21K smpl:3M ep:62K epch:12.54 loss:0.026 grdn:0.203 lr:7.7e-05 updt_s:1.716 data_s:1.655 smp/s:38 mem_gb:52.98
+Training:  36%|███▌      | 21600/60000 [18:49:49<33:47:19,  3.17s/step]INFO 2026-07-18 10:33:32 ot_train.py:649 step:22K smpl:3M ep:62K epch:12.66 loss:0.025 grdn:0.205 lr:7.6e-05 updt_s:1.686 data_s:1.659 smp/s:38 mem_gb:52.90
+Training:  36%|███▋      | 21800/60000 [19:00:53<33:54:20,  3.20s/step]INFO 2026-07-18 10:44:37 ot_train.py:649 step:22K smpl:3M ep:63K epch:12.78 loss:0.024 grdn:0.197 lr:7.6e-05 updt_s:1.688 data_s:1.630 smp/s:39 mem_gb:52.98
+Training:  37%|███▋      | 22000/60000 [19:11:23<34:08:59,  3.24s/step]INFO 2026-07-18 10:55:06 ot_train.py:649 step:22K smpl:3M ep:64K epch:12.90 loss:0.025 grdn:0.206 lr:7.5e-05 updt_s:1.531 data_s:1.611 smp/s:41 mem_gb:52.98
+Training:  37%|███▋      | 22200/60000 [19:22:17<33:10:05,  3.16s/step]INFO 2026-07-18 11:06:00 ot_train.py:649 step:22K smpl:3M ep:64K epch:13.01 loss:0.024 grdn:0.198 lr:7.5e-05 updt_s:1.692 data_s:1.573 smp/s:39 mem_gb:52.98
+Training:  37%|███▋      | 22400/60000 [19:32:50<35:15:18,  3.38s/step]INFO 2026-07-18 11:16:34 ot_train.py:649 step:22K smpl:3M ep:65K epch:13.13 loss:0.025 grdn:0.206 lr:7.4e-05 updt_s:1.577 data_s:1.583 smp/s:40 mem_gb:52.98
+Training:  38%|███▊      | 22600/60000 [19:43:25<32:52:00,  3.16s/step]INFO 2026-07-18 11:27:08 ot_train.py:649 step:23K smpl:3M ep:65K epch:13.25 loss:0.025 grdn:0.195 lr:7.4e-05 updt_s:1.553 data_s:1.614 smp/s:40 mem_gb:52.90
+Training:  38%|███▊      | 22800/60000 [19:53:48<32:10:01,  3.11s/step]INFO 2026-07-18 11:37:31 ot_train.py:649 step:23K smpl:3M ep:66K epch:13.36 loss:0.022 grdn:0.185 lr:7.3e-05 updt_s:1.534 data_s:1.573 smp/s:41 mem_gb:52.98
+Training:  38%|███▊      | 23000/60000 [20:04:17<32:10:41,  3.13s/step]INFO 2026-07-18 11:48:00 ot_train.py:649 step:23K smpl:3M ep:66K epch:13.48 loss:0.022 grdn:0.181 lr:7.3e-05 updt_s:1.532 data_s:1.607 smp/s:41 mem_gb:52.98
+Training:  39%|███▊      | 23200/60000 [20:14:40<32:00:59,  3.13s/step]INFO 2026-07-18 11:58:23 ot_train.py:649 step:23K smpl:3M ep:67K epch:13.60 loss:0.023 grdn:0.185 lr:7.2e-05 updt_s:1.522 data_s:1.587 smp/s:41 mem_gb:52.98
+Training:  39%|███▉      | 23400/60000 [20:25:00<31:09:37,  3.06s/step]INFO 2026-07-18 12:08:43 ot_train.py:649 step:23K smpl:3M ep:68K epch:13.72 loss:0.023 grdn:0.189 lr:7.2e-05 updt_s:1.523 data_s:1.574 smp/s:41 mem_gb:52.98
+Training:  39%|███▉      | 23600/60000 [20:35:26<31:33:36,  3.12s/step]INFO 2026-07-18 12:19:09 ot_train.py:649 step:24K smpl:3M ep:68K epch:13.83 loss:0.022 grdn:0.177 lr:7.1e-05 updt_s:1.543 data_s:1.582 smp/s:41 mem_gb:52.98
+Training:  40%|███▉      | 23800/60000 [20:45:50<30:33:28,  3.04s/step]INFO 2026-07-18 12:29:33 ot_train.py:649 step:24K smpl:3M ep:69K epch:13.95 loss:0.022 grdn:0.191 lr:7.1e-05 updt_s:1.544 data_s:1.570 smp/s:41 mem_gb:52.90
+Training:  40%|████      | 24000/60000 [20:56:12<31:07:24,  3.11s/step]INFO 2026-07-18 12:39:56 ot_train.py:649 step:24K smpl:3M ep:69K epch:14.07 loss:0.022 grdn:0.181 lr:7.0e-05 updt_s:1.527 data_s:1.579 smp/s:41 mem_gb:52.98
+Training:  40%|████      | 24200/60000 [21:06:30<30:08:46,  3.03s/step]INFO 2026-07-18 12:50:14 ot_train.py:649 step:24K smpl:3M ep:70K epch:14.19 loss:0.022 grdn:0.185 lr:7.0e-05 updt_s:1.595 data_s:1.489 smp/s:41 mem_gb:52.98
+Training:  41%|████      | 24400/60000 [21:16:53<29:42:08,  3.00s/step]INFO 2026-07-18 13:00:36 ot_train.py:649 step:24K smpl:3M ep:71K epch:14.30 loss:0.022 grdn:0.185 lr:6.9e-05 updt_s:1.542 data_s:1.565 smp/s:41 mem_gb:52.98
+Training:  41%|████      | 24600/60000 [21:27:17<30:20:05,  3.08s/step]INFO 2026-07-18 13:11:00 ot_train.py:649 step:25K smpl:3M ep:71K epch:14.42 loss:0.022 grdn:0.176 lr:6.9e-05 updt_s:1.524 data_s:1.589 smp/s:41 mem_gb:52.98
+Training:  41%|████▏     | 24800/60000 [21:37:40<30:42:23,  3.14s/step]INFO 2026-07-18 13:21:23 ot_train.py:649 step:25K smpl:3M ep:72K epch:14.54 loss:0.022 grdn:0.179 lr:6.8e-05 updt_s:1.556 data_s:1.553 smp/s:41 mem_gb:52.98
+Training:  42%|████▏     | 25000/60000 [21:48:07<31:01:14,  3.19s/step]INFO 2026-07-18 13:31:50 ot_train.py:649 step:25K smpl:3M ep:72K epch:14.65 loss:0.021 grdn:0.186 lr:6.8e-05 updt_s:1.522 data_s:1.607 smp/s:41 mem_gb:52.90
+Training:  42%|████▏     | 25200/60000 [21:58:26<29:10:10,  3.02s/step]INFO 2026-07-18 13:42:09 ot_train.py:649 step:25K smpl:3M ep:73K epch:14.77 loss:0.020 grdn:0.176 lr:6.7e-05 updt_s:1.559 data_s:1.530 smp/s:41 mem_gb:52.98
+Training:  42%|████▏     | 25400/60000 [22:08:44<33:08:30,  3.45s/step]INFO 2026-07-18 13:52:28 ot_train.py:649 step:25K smpl:3M ep:73K epch:14.89 loss:0.020 grdn:0.172 lr:6.7e-05 updt_s:1.572 data_s:1.515 smp/s:41 mem_gb:52.98
+Training:  43%|████▎     | 25600/60000 [22:18:53<31:21:02,  3.28s/step]INFO 2026-07-18 14:02:36 ot_train.py:649 step:26K smpl:3M ep:74K epch:15.01 loss:0.021 grdn:0.175 lr:6.6e-05 updt_s:1.636 data_s:1.400 smp/s:42 mem_gb:52.98
+Training:  43%|████▎     | 25800/60000 [22:29:08<28:46:44,  3.03s/step]INFO 2026-07-18 14:12:51 ot_train.py:649 step:26K smpl:3M ep:75K epch:15.12 loss:0.020 grdn:0.166 lr:6.6e-05 updt_s:1.635 data_s:1.437 smp/s:42 mem_gb:52.98
+Training:  43%|████▎     | 26000/60000 [22:39:10<28:50:09,  3.05s/step]INFO 2026-07-18 14:22:53 ot_train.py:649 step:26K smpl:3M ep:75K epch:15.24 loss:0.020 grdn:0.182 lr:6.5e-05 updt_s:1.620 data_s:1.384 smp/s:43 mem_gb:52.90
+Training:  44%|████▎     | 26200/60000 [22:49:09<26:53:51,  2.86s/step]INFO 2026-07-18 14:32:52 ot_train.py:649 step:26K smpl:3M ep:76K epch:15.36 loss:0.020 grdn:0.178 lr:6.5e-05 updt_s:1.623 data_s:1.367 smp/s:43 mem_gb:52.98
+Training:  44%|████▍     | 26400/60000 [22:59:18<29:13:10,  3.13s/step]INFO 2026-07-18 14:43:01 ot_train.py:649 step:26K smpl:3M ep:76K epch:15.47 loss:0.020 grdn:0.178 lr:6.4e-05 updt_s:1.597 data_s:1.441 smp/s:42 mem_gb:52.98
+Training:  44%|████▍     | 26600/60000 [23:09:36<28:15:51,  3.05s/step]INFO 2026-07-18 14:53:19 ot_train.py:649 step:27K smpl:3M ep:77K epch:15.59 loss:0.019 grdn:0.168 lr:6.4e-05 updt_s:1.564 data_s:1.520 smp/s:42 mem_gb:52.98
+Training:  45%|████▍     | 26800/60000 [23:19:56<27:49:29,  3.02s/step]INFO 2026-07-18 15:03:40 ot_train.py:649 step:27K smpl:3M ep:77K epch:15.71 loss:0.019 grdn:0.157 lr:6.3e-05 updt_s:1.551 data_s:1.547 smp/s:41 mem_gb:52.98
+Training:  45%|████▌     | 27000/60000 [23:30:25<28:16:40,  3.08s/step]INFO 2026-07-18 15:14:08 ot_train.py:649 step:27K smpl:3M ep:78K epch:15.83 loss:0.020 grdn:0.177 lr:6.3e-05 updt_s:1.605 data_s:1.533 smp/s:41 mem_gb:52.98
+Training:  45%|████▌     | 27200/60000 [23:40:48<31:58:46,  3.51s/step]INFO 2026-07-18 15:24:31 ot_train.py:649 step:27K smpl:3M ep:79K epch:15.94 loss:0.020 grdn:0.175 lr:6.2e-05 updt_s:1.577 data_s:1.531 smp/s:41 mem_gb:52.90
+Training:  46%|████▌     | 27400/60000 [23:51:00<26:52:15,  2.97s/step]INFO 2026-07-18 15:34:43 ot_train.py:649 step:27K smpl:4M ep:79K epch:16.06 loss:0.018 grdn:0.159 lr:6.1e-05 updt_s:1.594 data_s:1.460 smp/s:42 mem_gb:52.98
+Training:  46%|████▌     | 27600/60000 [24:01:01<26:51:57,  2.99s/step]INFO 2026-07-18 15:44:45 ot_train.py:649 step:28K smpl:4M ep:80K epch:16.18 loss:0.019 grdn:0.158 lr:6.1e-05 updt_s:1.597 data_s:1.406 smp/s:43 mem_gb:52.98
+Training:  46%|████▋     | 27800/60000 [24:11:01<27:49:08,  3.11s/step]INFO 2026-07-18 15:54:44 ot_train.py:649 step:28K smpl:4M ep:80K epch:16.30 loss:0.018 grdn:0.161 lr:6.0e-05 updt_s:1.634 data_s:1.359 smp/s:43 mem_gb:52.98
+Training:  47%|████▋     | 28000/60000 [24:21:02<30:44:19,  3.46s/step]INFO 2026-07-18 16:04:45 ot_train.py:649 step:28K smpl:4M ep:81K epch:16.41 loss:0.019 grdn:0.170 lr:6.0e-05 updt_s:1.617 data_s:1.384 smp/s:43 mem_gb:52.98
+Training:  47%|████▋     | 28200/60000 [24:31:08<27:41:48,  3.14s/step]INFO 2026-07-18 16:14:51 ot_train.py:649 step:28K smpl:4M ep:81K epch:16.53 loss:0.018 grdn:0.157 lr:5.9e-05 updt_s:1.613 data_s:1.409 smp/s:42 mem_gb:52.98
+Training:  47%|████▋     | 28400/60000 [24:41:15<30:45:19,  3.50s/step]INFO 2026-07-18 16:24:58 ot_train.py:649 step:28K smpl:4M ep:82K epch:16.65 loss:0.017 grdn:0.150 lr:5.9e-05 updt_s:1.632 data_s:1.398 smp/s:42 mem_gb:52.90
+Training:  48%|████▊     | 28600/60000 [24:51:30<25:33:53,  2.93s/step]INFO 2026-07-18 16:35:13 ot_train.py:649 step:29K smpl:4M ep:83K epch:16.76 loss:0.017 grdn:0.154 lr:5.8e-05 updt_s:1.617 data_s:1.453 smp/s:42 mem_gb:52.98
+Training:  48%|████▊     | 28800/60000 [25:01:40<26:48:11,  3.09s/step]INFO 2026-07-18 16:45:23 ot_train.py:649 step:29K smpl:4M ep:83K epch:16.88 loss:0.017 grdn:0.156 lr:5.8e-05 updt_s:1.665 data_s:1.381 smp/s:42 mem_gb:52.98
+Training:  48%|████▊     | 29000/60000 [25:11:43<23:28:50,  2.73s/step]INFO 2026-07-18 16:55:26 ot_train.py:649 step:29K smpl:4M ep:84K epch:17.00 loss:0.017 grdn:0.153 lr:5.7e-05 updt_s:1.587 data_s:1.421 smp/s:43 mem_gb:52.98
+Training:  49%|████▊     | 29200/60000 [25:22:05<25:10:19,  2.94s/step]INFO 2026-07-18 17:05:48 ot_train.py:649 step:29K smpl:4M ep:84K epch:17.12 loss:0.017 grdn:0.150 lr:5.7e-05 updt_s:1.577 data_s:1.529 smp/s:41 mem_gb:52.98
+Training:  49%|████▉     | 29400/60000 [25:32:24<24:29:39,  2.88s/step]INFO 2026-07-18 17:16:08 ot_train.py:649 step:29K smpl:4M ep:85K epch:17.23 loss:0.017 grdn:0.151 lr:5.6e-05 updt_s:1.608 data_s:1.484 smp/s:41 mem_gb:52.90
+Training:  49%|████▉     | 29600/60000 [25:42:43<27:16:22,  3.23s/step]INFO 2026-07-18 17:26:26 ot_train.py:649 step:30K smpl:4M ep:86K epch:17.35 loss:0.016 grdn:0.143 lr:5.5e-05 updt_s:1.671 data_s:1.418 smp/s:41 mem_gb:52.98
+Training:  50%|████▉     | 29800/60000 [25:52:47<23:37:49,  2.82s/step]INFO 2026-07-18 17:36:30 ot_train.py:649 step:30K smpl:4M ep:86K epch:17.47 loss:0.016 grdn:0.147 lr:5.5e-05 updt_s:1.611 data_s:1.401 smp/s:42 mem_gb:52.98
+Training:  50%|█████     | 30000/60000 [26:02:55<24:26:31,  2.93s/step]INFO 2026-07-18 17:46:38 ot_train.py:649 step:30K smpl:4M ep:87K epch:17.59 loss:0.017 grdn:0.156 lr:5.4e-05 updt_s:1.632 data_s:1.404 smp/s:42 mem_gb:52.98
+Training:  50%|█████     | 30200/60000 [26:13:06<24:40:32,  2.98s/step]INFO 2026-07-18 17:56:49 ot_train.py:649 step:30K smpl:4M ep:87K epch:17.70 loss:0.016 grdn:0.147 lr:5.4e-05 updt_s:1.628 data_s:1.422 smp/s:42 mem_gb:52.98
+Training:  51%|█████     | 30400/60000 [26:23:20<25:30:18,  3.10s/step]INFO 2026-07-18 18:07:04 ot_train.py:649 step:30K smpl:4M ep:88K epch:17.82 loss:0.016 grdn:0.143 lr:5.3e-05 updt_s:1.635 data_s:1.431 smp/s:42 mem_gb:52.98
+Training:  51%|█████     | 30600/60000 [26:33:33<25:05:29,  3.07s/step]INFO 2026-07-18 18:17:16 ot_train.py:649 step:31K smpl:4M ep:88K epch:17.94 loss:0.017 grdn:0.166 lr:5.3e-05 updt_s:1.600 data_s:1.459 smp/s:42 mem_gb:52.90
+Training:  51%|█████▏    | 30800/60000 [26:43:49<23:12:18,  2.86s/step]INFO 2026-07-18 18:27:32 ot_train.py:649 step:31K smpl:4M ep:89K epch:18.05 loss:0.016 grdn:0.147 lr:5.2e-05 updt_s:1.586 data_s:1.486 smp/s:42 mem_gb:52.98
+Training:  52%|█████▏    | 31000/60000 [26:54:06<23:15:45,  2.89s/step]INFO 2026-07-18 18:37:49 ot_train.py:649 step:31K smpl:4M ep:90K epch:18.17 loss:0.015 grdn:0.138 lr:5.2e-05 updt_s:1.601 data_s:1.481 smp/s:42 mem_gb:52.98
+Training:  52%|█████▏    | 31200/60000 [27:04:24<26:58:43,  3.37s/step]INFO 2026-07-18 18:48:07 ot_train.py:649 step:31K smpl:4M ep:90K epch:18.29 loss:0.015 grdn:0.143 lr:5.1e-05 updt_s:1.663 data_s:1.422 smp/s:41 mem_gb:52.98
+Training:  52%|█████▏    | 31400/60000 [27:14:38<22:15:32,  2.80s/step]INFO 2026-07-18 18:58:22 ot_train.py:649 step:31K smpl:4M ep:91K epch:18.41 loss:0.015 grdn:0.147 lr:5.1e-05 updt_s:1.623 data_s:1.443 smp/s:42 mem_gb:52.98
+Training:  53%|█████▎    | 31600/60000 [27:24:31<22:08:29,  2.81s/step]INFO 2026-07-18 19:08:14 ot_train.py:649 step:32K smpl:4M ep:91K epch:18.52 loss:0.015 grdn:0.140 lr:5.0e-05 updt_s:1.592 data_s:1.363 smp/s:43 mem_gb:52.98
+Training:  53%|█████▎    | 31800/60000 [27:34:31<22:00:37,  2.81s/step]INFO 2026-07-18 19:18:14 ot_train.py:649 step:32K smpl:4M ep:92K epch:18.64 loss:0.015 grdn:0.136 lr:4.9e-05 updt_s:1.598 data_s:1.397 smp/s:43 mem_gb:52.90
+Training:  53%|█████▎    | 32000/60000 [27:44:31<23:34:45,  3.03s/step]INFO 2026-07-18 19:28:14 ot_train.py:649 step:32K smpl:4M ep:92K epch:18.76 loss:0.014 grdn:0.135 lr:4.9e-05 updt_s:1.622 data_s:1.374 smp/s:43 mem_gb:52.98
+Training:  54%|█████▎    | 32200/60000 [27:54:48<22:47:35,  2.95s/step]INFO 2026-07-18 19:38:31 ot_train.py:649 step:32K smpl:4M ep:93K epch:18.87 loss:0.014 grdn:0.136 lr:4.8e-05 updt_s:1.656 data_s:1.423 smp/s:42 mem_gb:52.98
+Training:  54%|█████▍    | 32400/60000 [28:04:46<22:54:32,  2.99s/step]INFO 2026-07-18 19:48:29 ot_train.py:649 step:32K smpl:4M ep:94K epch:18.99 loss:0.015 grdn:0.150 lr:4.8e-05 updt_s:1.631 data_s:1.353 smp/s:43 mem_gb:52.98
+Training:  54%|█████▍    | 32600/60000 [28:14:34<23:53:43,  3.14s/step]INFO 2026-07-18 19:58:17 ot_train.py:649 step:33K smpl:4M ep:94K epch:19.11 loss:0.015 grdn:0.139 lr:4.7e-05 updt_s:1.619 data_s:1.319 smp/s:44 mem_gb:52.98
+Training:  55%|█████▍    | 32800/60000 [28:24:34<22:02:27,  2.92s/step]INFO 2026-07-18 20:08:17 ot_train.py:649 step:33K smpl:4M ep:95K epch:19.23 loss:0.015 grdn:0.156 lr:4.7e-05 updt_s:1.615 data_s:1.376 smp/s:43 mem_gb:52.90
+Training:  55%|█████▌    | 33000/60000 [28:34:42<23:24:29,  3.12s/step]INFO 2026-07-18 20:18:25 ot_train.py:649 step:33K smpl:4M ep:95K epch:19.34 loss:0.014 grdn:0.133 lr:4.6e-05 updt_s:1.622 data_s:1.414 smp/s:42 mem_gb:52.98
+Training:  55%|█████▌    | 33200/60000 [28:44:43<22:18:39,  3.00s/step]INFO 2026-07-18 20:28:26 ot_train.py:649 step:33K smpl:4M ep:96K epch:19.46 loss:0.014 grdn:0.133 lr:4.6e-05 updt_s:1.595 data_s:1.404 smp/s:43 mem_gb:52.98
+Training:  56%|█████▌    | 33400/60000 [28:54:56<22:50:42,  3.09s/step]INFO 2026-07-18 20:38:39 ot_train.py:649 step:33K smpl:4M ep:97K epch:19.58 loss:0.014 grdn:0.140 lr:4.5e-05 updt_s:1.634 data_s:1.427 smp/s:42 mem_gb:52.98
+Training:  56%|█████▌    | 33600/60000 [29:05:07<22:44:09,  3.10s/step]INFO 2026-07-18 20:48:50 ot_train.py:649 step:34K smpl:4M ep:97K epch:19.70 loss:0.014 grdn:0.136 lr:4.4e-05 updt_s:1.609 data_s:1.441 smp/s:42 mem_gb:52.98
+Training:  56%|█████▋    | 33800/60000 [29:15:16<21:54:22,  3.01s/step]INFO 2026-07-18 20:58:59 ot_train.py:649 step:34K smpl:4M ep:98K epch:19.81 loss:0.014 grdn:0.129 lr:4.4e-05 updt_s:1.597 data_s:1.441 smp/s:42 mem_gb:52.98
+Training:  57%|█████▋    | 34000/60000 [29:25:17<21:42:54,  3.01s/step]INFO 2026-07-18 21:09:00 ot_train.py:649 step:34K smpl:4M ep:98K epch:19.93 loss:0.013 grdn:0.135 lr:4.3e-05 updt_s:1.642 data_s:1.358 smp/s:43 mem_gb:52.90
+Training:  57%|█████▋    | 34200/60000 [29:35:17<21:45:41,  3.04s/step]INFO 2026-07-18 21:19:00 ot_train.py:649 step:34K smpl:4M ep:99K epch:20.05 loss:0.013 grdn:0.132 lr:4.3e-05 updt_s:1.670 data_s:1.327 smp/s:43 mem_gb:52.98
+Training:  57%|█████▋    | 34400/60000 [29:45:14<20:42:46,  2.91s/step]INFO 2026-07-18 21:28:57 ot_train.py:649 step:34K smpl:4M ep:99K epch:20.16 loss:0.013 grdn:0.131 lr:4.2e-05 updt_s:1.715 data_s:1.263 smp/s:43 mem_gb:52.98
+Training:  58%|█████▊    | 34600/60000 [29:55:07<21:53:35,  3.10s/step]INFO 2026-07-18 21:38:50 ot_train.py:649 step:35K smpl:4M ep:100K epch:20.28 loss:0.013 grdn:0.130 lr:4.2e-05 updt_s:1.730 data_s:1.229 smp/s:43 mem_gb:52.98
+Training:  58%|█████▊    | 34800/60000 [30:05:05<21:14:24,  3.03s/step]INFO 2026-07-18 21:48:49 ot_train.py:649 step:35K smpl:4M ep:101K epch:20.40 loss:0.013 grdn:0.131 lr:4.1e-05 updt_s:1.778 data_s:1.212 smp/s:43 mem_gb:52.98
+Training:  58%|█████▊    | 35000/60000 [30:15:10<23:58:32,  3.45s/step]INFO 2026-07-18 21:58:53 ot_train.py:649 step:35K smpl:4M ep:101K epch:20.52 loss:0.013 grdn:0.136 lr:4.1e-05 updt_s:1.777 data_s:1.238 smp/s:42 mem_gb:52.90
+Training:  59%|█████▊    | 35200/60000 [30:25:13<20:13:21,  2.94s/step]INFO 2026-07-18 22:08:57 ot_train.py:649 step:35K smpl:5M ep:102K epch:20.63 loss:0.012 grdn:0.128 lr:4.0e-05 updt_s:1.805 data_s:1.210 smp/s:42 mem_gb:52.98
+Training:  59%|█████▉    | 35400/60000 [30:35:10<21:23:14,  3.13s/step]INFO 2026-07-18 22:18:54 ot_train.py:649 step:35K smpl:5M ep:102K epch:20.75 loss:0.012 grdn:0.123 lr:4.0e-05 updt_s:1.757 data_s:1.223 smp/s:43 mem_gb:52.98
+Training:  59%|█████▉    | 35600/60000 [30:45:11<20:24:36,  3.01s/step]INFO 2026-07-18 22:28:55 ot_train.py:649 step:36K smpl:5M ep:103K epch:20.87 loss:0.012 grdn:0.121 lr:3.9e-05 updt_s:1.799 data_s:1.200 smp/s:43 mem_gb:52.98
+Training:  60%|█████▉    | 35800/60000 [30:55:11<19:56:02,  2.97s/step]INFO 2026-07-18 22:38:54 ot_train.py:649 step:36K smpl:5M ep:103K epch:20.98 loss:0.012 grdn:0.128 lr:3.9e-05 updt_s:1.808 data_s:1.187 smp/s:43 mem_gb:52.98
+Training:  60%|██████    | 36000/60000 [31:05:14<18:40:41,  2.80s/step]INFO 2026-07-18 22:48:58 ot_train.py:649 step:36K smpl:5M ep:104K epch:21.10 loss:0.012 grdn:0.129 lr:3.8e-05 updt_s:1.765 data_s:1.247 smp/s:42 mem_gb:52.98
+Training:  60%|██████    | 36200/60000 [31:15:58<19:52:29,  3.01s/step]INFO 2026-07-18 22:59:42 ot_train.py:649 step:36K smpl:5M ep:105K epch:21.22 loss:0.012 grdn:0.131 lr:3.7e-05 updt_s:1.736 data_s:1.476 smp/s:40 mem_gb:52.90
+Training:  61%|██████    | 36400/60000 [31:26:40<20:14:44,  3.09s/step]INFO 2026-07-18 23:10:23 ot_train.py:649 step:36K smpl:5M ep:105K epch:21.34 loss:0.011 grdn:0.116 lr:3.7e-05 updt_s:1.758 data_s:1.445 smp/s:40 mem_gb:52.98
+Training:  61%|██████    | 36600/60000 [31:38:02<20:30:43,  3.16s/step]INFO 2026-07-18 23:21:45 ot_train.py:649 step:37K smpl:5M ep:106K epch:21.45 loss:0.011 grdn:0.124 lr:3.6e-05 updt_s:1.729 data_s:1.672 smp/s:38 mem_gb:52.98
+Training:  61%|██████▏   | 36800/60000 [31:48:46<20:27:04,  3.17s/step]INFO 2026-07-18 23:32:29 ot_train.py:649 step:37K smpl:5M ep:106K epch:21.57 loss:0.011 grdn:0.131 lr:3.6e-05 updt_s:1.608 data_s:1.608 smp/s:40 mem_gb:52.98
+Training:  62%|██████▏   | 37000/60000 [31:59:15<20:23:58,  3.19s/step]INFO 2026-07-18 23:42:58 ot_train.py:649 step:37K smpl:5M ep:107K epch:21.69 loss:0.011 grdn:0.121 lr:3.5e-05 updt_s:1.576 data_s:1.563 smp/s:41 mem_gb:52.98
+Training:  62%|██████▏   | 37200/60000 [32:09:45<19:56:25,  3.15s/step]INFO 2026-07-18 23:53:28 ot_train.py:649 step:37K smpl:5M ep:108K epch:21.81 loss:0.011 grdn:0.116 lr:3.5e-05 updt_s:1.556 data_s:1.587 smp/s:41 mem_gb:52.98
+Training:  62%|██████▏   | 37400/60000 [32:20:07<21:28:21,  3.42s/step]INFO 2026-07-19 00:03:50 ot_train.py:649 step:37K smpl:5M ep:108K epch:21.92 loss:0.011 grdn:0.123 lr:3.4e-05 updt_s:1.543 data_s:1.563 smp/s:41 mem_gb:52.90
+Training:  63%|██████▎   | 37600/60000 [32:30:21<19:39:11,  3.16s/step]INFO 2026-07-19 00:14:04 ot_train.py:649 step:38K smpl:5M ep:109K epch:22.04 loss:0.011 grdn:0.120 lr:3.4e-05 updt_s:1.545 data_s:1.518 smp/s:42 mem_gb:52.98
+Training:  63%|██████▎   | 37800/60000 [32:40:31<19:22:56,  3.14s/step]INFO 2026-07-19 00:24:14 ot_train.py:649 step:38K smpl:5M ep:109K epch:22.16 loss:0.011 grdn:0.119 lr:3.3e-05 updt_s:1.588 data_s:1.457 smp/s:42 mem_gb:52.98
+Training:  63%|██████▎   | 38000/60000 [32:50:39<18:09:30,  2.97s/step]INFO 2026-07-19 00:34:22 ot_train.py:649 step:38K smpl:5M ep:110K epch:22.27 loss:0.011 grdn:0.116 lr:3.3e-05 updt_s:1.664 data_s:1.373 smp/s:42 mem_gb:52.98
+Training:  64%|██████▎   | 38200/60000 [33:00:57<17:41:34,  2.92s/step]INFO 2026-07-19 00:44:40 ot_train.py:649 step:38K smpl:5M ep:110K epch:22.39 loss:0.010 grdn:0.115 lr:3.2e-05 updt_s:1.638 data_s:1.446 smp/s:42 mem_gb:52.98
+Training:  64%|██████▍   | 38400/60000 [33:11:02<18:44:42,  3.12s/step]INFO 2026-07-19 00:54:46 ot_train.py:649 step:38K smpl:5M ep:111K epch:22.51 loss:0.011 grdn:0.121 lr:3.2e-05 updt_s:1.614 data_s:1.409 smp/s:42 mem_gb:52.90
+Training:  64%|██████▍   | 38600/60000 [33:21:11<17:10:25,  2.89s/step]INFO 2026-07-19 01:04:54 ot_train.py:649 step:39K smpl:5M ep:112K epch:22.63 loss:0.010 grdn:0.110 lr:3.1e-05 updt_s:1.628 data_s:1.407 smp/s:42 mem_gb:52.98
+Training:  65%|██████▍   | 38800/60000 [33:31:16<17:08:54,  2.91s/step]INFO 2026-07-19 01:14:59 ot_train.py:649 step:39K smpl:5M ep:112K epch:22.74 loss:0.010 grdn:0.110 lr:3.1e-05 updt_s:1.618 data_s:1.401 smp/s:42 mem_gb:52.98
+Training:  65%|██████▌   | 39000/60000 [33:41:19<17:47:48,  3.05s/step]INFO 2026-07-19 01:25:02 ot_train.py:649 step:39K smpl:5M ep:113K epch:22.86 loss:0.010 grdn:0.116 lr:3.0e-05 updt_s:1.655 data_s:1.356 smp/s:43 mem_gb:52.98
+Training:  65%|██████▌   | 39200/60000 [33:51:30<18:04:05,  3.13s/step]INFO 2026-07-19 01:35:14 ot_train.py:649 step:39K smpl:5M ep:113K epch:22.98 loss:0.010 grdn:0.115 lr:3.0e-05 updt_s:1.616 data_s:1.436 smp/s:42 mem_gb:52.98
+Training:  66%|██████▌   | 39400/60000 [34:01:51<16:25:56,  2.87s/step]INFO 2026-07-19 01:45:35 ot_train.py:649 step:39K smpl:5M ep:114K epch:23.10 loss:0.010 grdn:0.117 lr:2.9e-05 updt_s:1.601 data_s:1.498 smp/s:41 mem_gb:52.98
+Training:  66%|██████▌   | 39600/60000 [34:11:50<16:58:14,  2.99s/step]INFO 2026-07-19 01:55:34 ot_train.py:649 step:40K smpl:5M ep:114K epch:23.21 loss:0.010 grdn:0.113 lr:2.9e-05 updt_s:1.563 data_s:1.426 smp/s:43 mem_gb:52.90
+Training:  66%|██████▋   | 39800/60000 [34:21:52<17:49:04,  3.18s/step]INFO 2026-07-19 02:05:35 ot_train.py:649 step:40K smpl:5M ep:115K epch:23.33 loss:0.009 grdn:0.110 lr:2.8e-05 updt_s:1.627 data_s:1.376 smp/s:43 mem_gb:52.98
+Training:  67%|██████▋   | 40000/60000 [34:31:36<15:55:02,  2.87s/step]INFO 2026-07-19 02:15:19 ot_train.py:649 step:40K smpl:5M ep:116K epch:23.45 loss:0.009 grdn:0.111 lr:2.8e-05 updt_s:1.621 data_s:1.293 smp/s:44 mem_gb:52.98
+Training:  67%|██████▋   | 40066/60000 [34:34:55<16:36:40,  3.00s/step]

Abs_6D + DeltaEE/_archive/interrupted_reboot/RKD_TimewarpVAE/RSCLstyle_cosine_60K.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-e5avxg6w/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_setup.py:_flush():81] Configure stats pid to 499773
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260717_154331-e5avxg6w/logs/debug.log
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260717_154331-e5avxg6w/logs/debug-internal.log
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:init():766] calling init triggers
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-17 15:43:31,453 INFO    MainThread:499773 [wandb_init.py:init():814] starting backend
+2026-07-17 15:43:31,678 INFO    MainThread:499773 [wandb_init.py:init():829] sending inform_init request
+2026-07-17 15:43:32,380 INFO    MainThread:499773 [wandb_init.py:init():834] backend started and connected
+2026-07-17 15:43:32,384 INFO    MainThread:499773 [wandb_init.py:init():904] updated telemetry
+2026-07-17 15:43:32,392 INFO    MainThread:499773 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-17 15:43:32,831 INFO    MainThread:499773 [wandb_init.py:init():972] starting run threads in backend
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-17 15:43:33,011 INFO    MainThread:499773 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-17 15:43:33,016 INFO    MainThread:499773 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/Baseline/wandb/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-07-19T23:04:31.813242533+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-19T23:04:31.81382052+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-19T23:04:32.344730077+09:00","level":"INFO","msg":"stream: created new stream","id":"ajlv9qc5"}
+{"time":"2026-07-19T23:04:32.344968538+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-19T23:04:32.345008264+09:00","level":"INFO","msg":"writer: started","stream_id":"ajlv9qc5"}
+{"time":"2026-07-19T23:04:32.34508556+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-19T23:04:32.345059474+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-19T23:04:32.860950364+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-19T23:04:33.243629437+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:04:47.86138057+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":15,"uploaded_len":2}
+{"time":"2026-07-19T23:04:48.18030455+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:02.861828226+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:03.262381224+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:17.861165647+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:18.148957301+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:32.861865868+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:33.23129356+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:47.861003074+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:48.246172948+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:02.86159547+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":9,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:03.269857393+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:17.861688399+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:18.289365504+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:32.861609778+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":13,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:33.315776295+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:47.861626977+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":15,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:48.362124826+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:02.861392553+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":17,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:03.328352457+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:17.860930238+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":19,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:18.180524239+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:32.861289033+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":21,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:33.243317934+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:47.861114611+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":23,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:48.165733874+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:02.861065838+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":25,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:03.266581507+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:17.861924025+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":27,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:18.263160805+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:32.861265948+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":29,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:33.221054916+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:47.861669879+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":31,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:48.256566549+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:02.861346575+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":33,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:03.287396886+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:17.861498281+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":35,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:18.295465227+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:32.861265597+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:33.31683242+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:47.861651377+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":39,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:48.27119445+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:02.861398324+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":41,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:03.248283028+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:17.861807031+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":43,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:18.299030334+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:32.861482457+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":45,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:33.270874405+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:47.860914999+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":47,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:48.152543457+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:02.860838398+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":49,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:03.231975975+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:17.861078604+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":51,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:18.196012531+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:32.861061466+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":53,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:33.283575595+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:47.86093486+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":55,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:48.266688556+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:02.861659187+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":57,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:03.237623334+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:17.861674818+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":59,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:18.287218148+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:32.861204007+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":61,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:33.335261363+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:47.861786253+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":63,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:48.129544067+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:02.861912949+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":65,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:03.229115242+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:17.861389905+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":67,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:18.15830388+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:32.861324836+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":69,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:33.296722029+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:47.861380357+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":71,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:48.223402479+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:02.861782715+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":73,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:03.208286934+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:17.860889195+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":75,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:18.181581208+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:32.861876269+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":77,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:33.282595209+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/Baseline/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_setup.py:_flush():81] Configure stats pid to 2036848
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline/wandb/run-20260719_230431-ajlv9qc5/logs/debug.log
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline/wandb/run-20260719_230431-ajlv9qc5/logs/debug-internal.log
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:init():766] calling init triggers
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:init():814] starting backend
+2026-07-19 23:04:31,802 INFO    MainThread:2036848 [wandb_init.py:init():829] sending inform_init request
+2026-07-19 23:04:32,346 INFO    MainThread:2036848 [wandb_init.py:init():834] backend started and connected
+2026-07-19 23:04:32,348 INFO    MainThread:2036848 [wandb_init.py:init():904] updated telemetry
+2026-07-19 23:04:32,356 INFO    MainThread:2036848 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-19 23:04:32,748 INFO    MainThread:2036848 [wandb_init.py:init():972] starting run threads in backend
+2026-07-19 23:04:32,856 INFO    MainThread:2036848 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-19 23:04:32,857 INFO    MainThread:2036848 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-19 23:04:32,857 INFO    MainThread:2036848 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-19 23:04:32,857 INFO    MainThread:2036848 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-19 23:04:32,859 INFO    MainThread:2036848 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/Baseline/wandb/run-20260719_230431-ajlv9qc5/files/wandb-metadata.json

@@ -0,0 +1,91 @@
+{
+  "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-19T14:04:31.585716Z",
+  "args": [
+    "--config_path=/tmp/insight_lerobot_cfg_vk2mvoqq/baseline.yaml",
+    "--batch_size=128",
+    "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a9df08f7da2d8eb1fa7f28708b4eb6f6dc81b1f9"
+  },
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline",
+  "host": "DGX-H200-01",
+  "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "813929263104"
+    }
+  },
+  "memory": {
+    "total": "2164177309696"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "2vsqnxc7lgikyhb4d9fwkd4x9qym3rb5"
+}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/Baseline/wandb/run-20260719_230431-ajlv9qc5/logs/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-07-19T23:04:31.813242533+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-19T23:04:31.81382052+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-19T23:04:32.344730077+09:00","level":"INFO","msg":"stream: created new stream","id":"ajlv9qc5"}
+{"time":"2026-07-19T23:04:32.344968538+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-19T23:04:32.345008264+09:00","level":"INFO","msg":"writer: started","stream_id":"ajlv9qc5"}
+{"time":"2026-07-19T23:04:32.34508556+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-19T23:04:32.345059474+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-19T23:04:32.860950364+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-19T23:04:33.243629437+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:04:47.86138057+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":15,"uploaded_len":2}
+{"time":"2026-07-19T23:04:48.18030455+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:02.861828226+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:03.262381224+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:17.861165647+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:18.148957301+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:32.861865868+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:33.23129356+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:47.861003074+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:48.246172948+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:02.86159547+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":9,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:03.269857393+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:17.861688399+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:18.289365504+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:32.861609778+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":13,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:33.315776295+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:47.861626977+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":15,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:48.362124826+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:02.861392553+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":17,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:03.328352457+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:17.860930238+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":19,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:18.180524239+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:32.861289033+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":21,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:33.243317934+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:47.861114611+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":23,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:48.165733874+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:02.861065838+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":25,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:03.266581507+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:17.861924025+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":27,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:18.263160805+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:32.861265948+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":29,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:33.221054916+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:47.861669879+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":31,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:48.256566549+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:02.861346575+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":33,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:03.287396886+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:17.861498281+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":35,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:18.295465227+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:32.861265597+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:33.31683242+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:47.861651377+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":39,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:48.27119445+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:02.861398324+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":41,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:03.248283028+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:17.861807031+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":43,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:18.299030334+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:32.861482457+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":45,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:33.270874405+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:47.860914999+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":47,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:48.152543457+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:02.860838398+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":49,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:03.231975975+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:17.861078604+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":51,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:18.196012531+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:32.861061466+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":53,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:33.283575595+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:47.86093486+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":55,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:48.266688556+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:02.861659187+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":57,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:03.237623334+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:17.861674818+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":59,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:18.287218148+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:32.861204007+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":61,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:33.335261363+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:47.861786253+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":63,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:48.129544067+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:02.861912949+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":65,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:03.229115242+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:17.861389905+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":67,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:18.15830388+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:32.861324836+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":69,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:33.296722029+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:47.861380357+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":71,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:48.223402479+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:02.861782715+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":73,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:03.208286934+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:17.860889195+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":75,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:18.181581208+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:32.861876269+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":77,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:33.282595209+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/Baseline/wandb/run-20260719_230431-ajlv9qc5/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_setup.py:_flush():81] Configure stats pid to 2036848
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline/wandb/run-20260719_230431-ajlv9qc5/logs/debug.log
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline/wandb/run-20260719_230431-ajlv9qc5/logs/debug-internal.log
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:init():766] calling init triggers
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-19 23:04:31,586 INFO    MainThread:2036848 [wandb_init.py:init():814] starting backend
+2026-07-19 23:04:31,802 INFO    MainThread:2036848 [wandb_init.py:init():829] sending inform_init request
+2026-07-19 23:04:32,346 INFO    MainThread:2036848 [wandb_init.py:init():834] backend started and connected
+2026-07-19 23:04:32,348 INFO    MainThread:2036848 [wandb_init.py:init():904] updated telemetry
+2026-07-19 23:04:32,356 INFO    MainThread:2036848 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-19 23:04:32,748 INFO    MainThread:2036848 [wandb_init.py:init():972] starting run threads in backend
+2026-07-19 23:04:32,856 INFO    MainThread:2036848 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-19 23:04:32,857 INFO    MainThread:2036848 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-19 23:04:32,857 INFO    MainThread:2036848 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-19 23:04:32,857 INFO    MainThread:2036848 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-19 23:04:32,859 INFO    MainThread:2036848 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-07-19T23:04:35.960247185+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-19T23:04:35.960754745+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-19T23:04:36.639007187+09:00","level":"INFO","msg":"stream: created new stream","id":"7ob3zmbd"}
+{"time":"2026-07-19T23:04:36.639131622+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-19T23:04:36.639327255+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-19T23:04:36.639379585+09:00","level":"INFO","msg":"writer: started","stream_id":"7ob3zmbd"}
+{"time":"2026-07-19T23:04:36.63950064+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-19T23:04:37.267802748+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-19T23:04:37.751082168+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:04:52.268769544+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":15,"uploaded_len":2}
+{"time":"2026-07-19T23:04:52.573885823+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:07.268389868+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-07-19T23:05:07.606875177+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:22.268050665+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:22.715586991+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:37.268124734+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:37.645658129+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:52.268366171+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:52.557736918+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:07.268106669+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:07.621886852+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:22.268682413+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:22.726555788+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:37.268359275+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:37.990790739+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:52.268295041+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:52.682552499+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:07.268549568+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":18,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:07.673104633+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:22.268615978+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:22.683091689+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:37.268682698+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:37.681312757+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:52.268339864+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":23,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:52.640850521+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:07.268194607+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":26,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:07.702499163+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:22.268328372+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:22.715021671+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:37.268193555+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:37.626022328+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:52.267803239+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":32,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:52.672064821+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:07.268317912+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:07.74037807+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:22.267592835+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":36,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:22.710706328+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:37.268099216+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:37.700480492+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:52.268238695+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:52.69520669+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:07.268072235+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":42,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:07.662257161+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:22.268470825+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:22.686271986+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:37.268174148+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:37.575139887+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:52.268481192+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":48,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:52.595653047+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:07.268211489+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:07.628512238+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:22.267761494+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":51,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:22.673196918+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:37.267983859+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":54,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:37.658592212+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:52.26826316+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":56,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:52.735918521+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:07.267639278+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":58,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:07.654674405+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:22.26833269+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":59,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:22.711237184+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:37.268231295+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":62,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:37.740143018+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:52.268083109+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":63,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:52.728951786+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:07.267809308+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":66,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:07.573546038+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:22.267745129+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:22.577801422+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:37.26817292+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":70,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:37.709547258+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:52.268335918+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":72,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:52.716685388+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:07.268130681+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":74,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:07.706210103+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:22.268283925+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":76,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:22.570987679+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:37.268090837+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":78,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:37.638228428+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_setup.py:_flush():81] Configure stats pid to 2036850
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/logs/debug.log
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/logs/debug-internal.log
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:init():766] calling init triggers
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:init():814] starting backend
+2026-07-19 23:04:35,947 INFO    MainThread:2036850 [wandb_init.py:init():829] sending inform_init request
+2026-07-19 23:04:36,641 INFO    MainThread:2036850 [wandb_init.py:init():834] backend started and connected
+2026-07-19 23:04:36,645 INFO    MainThread:2036850 [wandb_init.py:init():904] updated telemetry
+2026-07-19 23:04:36,657 INFO    MainThread:2036850 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-19 23:04:37,058 INFO    MainThread:2036850 [wandb_init.py:init():972] starting run threads in backend
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-19 23:04:37,264 INFO    MainThread:2036850 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/files/output.log

@@ -0,0 +1,17 @@
+INFO 2026-07-19 23:04:37 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-19 23:04:37 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/7ob3zmbd
+INFO 2026-07-19 23:04:37 ot_train.py:298 Creating dataset
+INFO 2026-07-19 23:04:41 ot_train.py:332 Creating policy
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 2422.64it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 2053.55it/s]
+INFO 2026-07-19 23:04:46 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-19 23:04:46 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K
+INFO 2026-07-19 23:04:46 ot_train.py:446 cfg.steps=60000 (60K)
+INFO 2026-07-19 23:04:46 ot_train.py:447 dataset.num_frames=218367 (218K)
+INFO 2026-07-19 23:04:46 ot_train.py:448 dataset.num_episodes=4930
+INFO 2026-07-19 23:04:46 ot_train.py:451 Effective batch size: 128 x 1 = 128
+INFO 2026-07-19 23:04:46 ot_train.py:452 num_learnable_params=1622090881 (2B)
+INFO 2026-07-19 23:04:46 ot_train.py:453 num_total_params=3145712033 (3B)
+Training:   0%|          | 0/60000 [00:00<?, ?step/s]INFO 2026-07-19 23:04:47 ot_train.py:604 Start offline training on a fixed dataset, with effective batch size: 128
+Training:   0%|          | 173/60000 [09:45<63:52:34,  3.84s/step]

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/files/requirements.txt

@@ -0,0 +1,117 @@
+pyarrow==25.0.0
+importlib_metadata==9.0.0
+wheel==0.47.0
+cloudpickle==3.1.2
+nvidia-cusolver-cu12==11.7.3.90
+multiprocess==0.70.19
+torch==2.11.0+cu128
+nvidia-nvjitlink-cu12==12.8.93
+fsspec==2026.2.0
+pytz==2026.2
+hf-xet==1.5.1
+tokenizers==0.22.2
+opencv-python-headless==4.13.0.92
+cuda-pathfinder==1.2.2
+frozenlist==1.8.0
+smmap==5.0.3
+httpcore==1.0.9
+nvidia-nvshmem-cu12==3.4.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+annotated-types==0.7.0
+platformdirs==4.10.0
+click==8.4.2
+pyyaml-include==1.4.1
+MarkupSafe==3.0.3
+Jinja2==3.1.6
+urllib3==2.7.0
+nvidia-curand-cu12==10.3.9.90
+certifi==2026.6.17
+nvidia-cuda-runtime-cu12==12.8.90
+torchcodec==0.11.1
+toml==0.10.2
+av==15.1.0
+nvidia-cufft-cu12==11.3.3.83
+attrs==26.1.0
+lerobot==0.6.0
+PyYAML==6.0.3
+safetensors==0.8.0
+diffusers==0.35.2
+requests==2.34.2
+nvidia-nvtx-cu12==12.8.90
+charset-normalizer==3.4.9
+networkx==3.6.1
+cmake==4.1.3
+pydantic_core==2.46.4
+timm==1.0.28
+dm-tree==0.1.10
+nvidia-cufile-cu12==1.13.1.3
+termcolor==3.3.0
+multidict==6.7.1
+filelock==3.29.0
+tzdata==2026.3
+aiohappyeyeballs==2.7.1
+httpx==0.28.1
+nvidia-nccl-cu12==2.28.9
+Pygments==2.20.0
+einops==0.8.2
+nvidia-cusparse-cu12==12.5.8.93
+torchvision==0.26.0+cu128
+dill==0.4.1
+GitPython==3.1.51
+nvidia-cuda-cupti-cu12==12.8.90
+decord==0.6.0
+sympy==1.14.0
+cuda-toolkit==12.8.1
+python-dateutil==2.9.0.post0
+shellingham==1.5.4
+propcache==0.5.2
+Farama-Notifications==0.0.6
+pillow==12.2.0
+cuda-bindings==12.9.4
+huggingface_hub==1.23.0
+pandas==2.3.3
+typing-inspect==0.9.0
+packaging==25.0
+rich==15.0.0
+nvidia-cusparselt-cu12==0.7.1
+setuptools==78.1.0
+wrapt==2.2.2
+mypy_extensions==1.1.0
+tqdm==4.68.4
+draccus==0.10.0
+aiosignal==1.4.0
+mpmath==1.3.0
+numpy==2.2.6
+typing_extensions==4.15.0
+psutil==7.2.2
+mdurl==0.1.2
+nvidia-cublas-cu12==12.8.4.1
+absl-py==2.5.0
+gitdb==4.0.12
+transformers==5.5.4
+accelerate==1.14.0
+pydantic==2.13.4
+pip==26.1.2
+markdown-it-py==4.2.0
+typer==0.26.8
+triton==3.6.0
+wandb==0.27.2
+regex==2026.7.10
+yarl==1.24.2
+jsonlines==4.0.0
+mergedeep==1.3.4
+aiohttp==3.14.1
+datasets==4.8.5
+gymnasium==1.3.0
+peft==0.19.1
+zipp==4.1.0
+idna==3.18
+nvidia-cudnn-cu12==9.19.0.56
+h11==0.16.0
+sentry-sdk==2.65.0
+annotated-doc==0.0.4
+protobuf==7.35.1
+typing-inspection==0.4.2
+anyio==4.14.2
+xxhash==3.8.1
+six==1.17.0

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/files/wandb-metadata.json

@@ -0,0 +1,91 @@
+{
+  "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-19T14:04:35.721770Z",
+  "args": [
+    "--config_path=/tmp/insight_lerobot_cfg_49uglsrc/LAPstyle_RKD_linear_decay_10K.yaml",
+    "--batch_size=128",
+    "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a9df08f7da2d8eb1fa7f28708b4eb6f6dc81b1f9"
+  },
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K",
+  "host": "DGX-H200-01",
+  "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "813929852928"
+    }
+  },
+  "memory": {
+    "total": "2164177309696"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "f3wbl1tz4dd8p8pxgdll61uby3zd2y4t"
+}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/logs/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-07-19T23:04:35.960247185+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-19T23:04:35.960754745+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-19T23:04:36.639007187+09:00","level":"INFO","msg":"stream: created new stream","id":"7ob3zmbd"}
+{"time":"2026-07-19T23:04:36.639131622+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-19T23:04:36.639327255+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-19T23:04:36.639379585+09:00","level":"INFO","msg":"writer: started","stream_id":"7ob3zmbd"}
+{"time":"2026-07-19T23:04:36.63950064+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-19T23:04:37.267802748+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-19T23:04:37.751082168+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:04:52.268769544+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":15,"uploaded_len":2}
+{"time":"2026-07-19T23:04:52.573885823+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:07.268389868+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-07-19T23:05:07.606875177+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:22.268050665+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:22.715586991+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:37.268124734+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:37.645658129+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:52.268366171+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:52.557736918+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:07.268106669+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:07.621886852+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:22.268682413+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:22.726555788+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:37.268359275+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:37.990790739+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:52.268295041+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:52.682552499+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:07.268549568+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":18,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:07.673104633+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:22.268615978+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:22.683091689+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:37.268682698+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:37.681312757+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:52.268339864+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":23,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:52.640850521+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:07.268194607+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":26,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:07.702499163+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:22.268328372+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:22.715021671+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:37.268193555+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:37.626022328+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:52.267803239+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":32,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:52.672064821+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:07.268317912+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:07.74037807+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:22.267592835+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":36,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:22.710706328+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:37.268099216+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:37.700480492+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:52.268238695+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:52.69520669+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:07.268072235+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":42,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:07.662257161+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:22.268470825+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:22.686271986+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:37.268174148+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:37.575139887+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:52.268481192+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":48,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:52.595653047+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:07.268211489+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:07.628512238+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:22.267761494+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":51,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:22.673196918+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:37.267983859+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":54,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:37.658592212+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:52.26826316+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":56,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:52.735918521+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:07.267639278+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":58,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:07.654674405+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:22.26833269+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":59,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:22.711237184+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:37.268231295+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":62,"events_lines":1,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:37.740143018+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:52.268083109+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":63,"events_lines":3,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:52.728951786+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:07.267809308+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":66,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:07.573546038+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:22.267745129+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:22.577801422+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:37.26817292+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":70,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:37.709547258+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:52.268335918+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":72,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:52.716685388+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:07.268130681+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":74,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:07.706210103+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:22.268283925+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":76,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:22.570987679+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:37.268090837+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":78,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:37.638228428+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_setup.py:_flush():81] Configure stats pid to 2036850
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/logs/debug.log
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/logs/debug-internal.log
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:init():766] calling init triggers
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-19 23:04:35,723 INFO    MainThread:2036850 [wandb_init.py:init():814] starting backend
+2026-07-19 23:04:35,947 INFO    MainThread:2036850 [wandb_init.py:init():829] sending inform_init request
+2026-07-19 23:04:36,641 INFO    MainThread:2036850 [wandb_init.py:init():834] backend started and connected
+2026-07-19 23:04:36,645 INFO    MainThread:2036850 [wandb_init.py:init():904] updated telemetry
+2026-07-19 23:04:36,657 INFO    MainThread:2036850 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-19 23:04:37,058 INFO    MainThread:2036850 [wandb_init.py:init():972] starting run threads in backend
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-19 23:04:37,260 INFO    MainThread:2036850 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-19 23:04:37,264 INFO    MainThread:2036850 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-07-19T23:04:35.840125776+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-19T23:04:35.840638556+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-19T23:04:36.54060383+09:00","level":"INFO","msg":"stream: created new stream","id":"bvj1u4hd"}
+{"time":"2026-07-19T23:04:36.54078243+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-19T23:04:36.540866846+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-19T23:04:36.540882054+09:00","level":"INFO","msg":"writer: started","stream_id":"bvj1u4hd"}
+{"time":"2026-07-19T23:04:36.540911877+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-19T23:04:37.360831701+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-19T23:04:37.768316877+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:04:52.361524384+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":15,"uploaded_len":2}
+{"time":"2026-07-19T23:04:52.611456834+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:07.361087657+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
+{"time":"2026-07-19T23:05:07.675579166+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:22.36129589+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:22.690724217+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:37.361504492+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:37.748248702+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:52.361539549+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:52.746890982+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:07.361761595+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":9,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:07.758189689+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:22.360814312+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:22.732027144+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:37.360944034+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":13,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:37.957533437+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:52.360824557+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":15,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:52.709354336+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:07.361770298+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":17,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:07.77667497+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:22.361532494+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":19,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:22.750366731+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:37.361366156+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":21,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:37.7047906+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:52.360964029+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":23,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:52.741271853+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:07.361007665+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":25,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:07.759270115+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:22.360784311+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":27,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:22.739878533+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:37.362067792+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":29,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:37.697247556+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:52.361783353+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":31,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:52.663210353+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:07.361805829+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":33,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:07.825541965+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:22.360992854+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":35,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:22.78458095+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:37.361584281+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:37.794040736+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:52.361131877+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":39,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:52.834839905+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:07.361007717+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":41,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:07.749488213+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:22.361854828+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":43,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:22.781090405+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:37.360963306+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":45,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:37.642609437+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:52.36137222+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":47,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:52.842313674+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:07.361553301+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":49,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:07.726615867+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:22.361762701+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":51,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:22.786651142+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:37.360846503+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":53,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:37.816027731+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:52.361149588+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":55,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:52.760242274+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:07.361205542+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":57,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:07.669311643+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:22.361560676+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":59,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:22.795167228+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:37.361547436+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":61,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:37.812676351+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:52.360836304+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":63,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:52.658061715+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:07.361209022+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":65,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:07.659851244+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:22.361327822+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":67,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:22.86967031+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:37.361324551+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":69,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:37.79306621+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:52.361469725+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":71,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:52.84583428+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:07.361146485+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":73,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:07.751834249+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:22.361067976+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":75,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:22.727458065+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:37.36095564+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":77,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:37.755069398+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_setup.py:_flush():81] Configure stats pid to 2036851
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/logs/debug.log
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/logs/debug-internal.log
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:init():766] calling init triggers
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:init():814] starting backend
+2026-07-19 23:04:35,830 INFO    MainThread:2036851 [wandb_init.py:init():829] sending inform_init request
+2026-07-19 23:04:36,541 INFO    MainThread:2036851 [wandb_init.py:init():834] backend started and connected
+2026-07-19 23:04:36,546 INFO    MainThread:2036851 [wandb_init.py:init():904] updated telemetry
+2026-07-19 23:04:36,557 INFO    MainThread:2036851 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-19 23:04:37,200 INFO    MainThread:2036851 [wandb_init.py:init():972] starting run threads in backend
+2026-07-19 23:04:37,351 INFO    MainThread:2036851 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-19 23:04:37,353 INFO    MainThread:2036851 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-19 23:04:37,353 INFO    MainThread:2036851 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-19 23:04:37,353 INFO    MainThread:2036851 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-19 23:04:37,357 INFO    MainThread:2036851 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/files/output.log

@@ -0,0 +1,17 @@
+INFO 2026-07-19 23:04:37 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-19 23:04:37 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/bvj1u4hd
+INFO 2026-07-19 23:04:37 ot_train.py:298 Creating dataset
+INFO 2026-07-19 23:04:41 ot_train.py:332 Creating policy
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 4662.26it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1751.88it/s]
+INFO 2026-07-19 23:04:46 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-19 23:04:46 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K
+INFO 2026-07-19 23:04:46 ot_train.py:446 cfg.steps=60000 (60K)
+INFO 2026-07-19 23:04:46 ot_train.py:447 dataset.num_frames=218367 (218K)
+INFO 2026-07-19 23:04:46 ot_train.py:448 dataset.num_episodes=4930
+INFO 2026-07-19 23:04:46 ot_train.py:451 Effective batch size: 128 x 1 = 128
+INFO 2026-07-19 23:04:46 ot_train.py:452 num_learnable_params=1622090881 (2B)
+INFO 2026-07-19 23:04:46 ot_train.py:453 num_total_params=3145712033 (3B)
+Training:   0%|          | 0/60000 [00:00<?, ?step/s]INFO 2026-07-19 23:04:47 ot_train.py:604 Start offline training on a fixed dataset, with effective batch size: 128
+Training:   0%|          | 176/60000 [09:49<66:34:12,  4.01s/step]

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/files/requirements.txt

@@ -0,0 +1,117 @@
+pyarrow==25.0.0
+importlib_metadata==9.0.0
+wheel==0.47.0
+cloudpickle==3.1.2
+nvidia-cusolver-cu12==11.7.3.90
+multiprocess==0.70.19
+torch==2.11.0+cu128
+nvidia-nvjitlink-cu12==12.8.93
+fsspec==2026.2.0
+pytz==2026.2
+hf-xet==1.5.1
+tokenizers==0.22.2
+opencv-python-headless==4.13.0.92
+cuda-pathfinder==1.2.2
+frozenlist==1.8.0
+smmap==5.0.3
+httpcore==1.0.9
+nvidia-nvshmem-cu12==3.4.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+annotated-types==0.7.0
+platformdirs==4.10.0
+click==8.4.2
+pyyaml-include==1.4.1
+MarkupSafe==3.0.3
+Jinja2==3.1.6
+urllib3==2.7.0
+nvidia-curand-cu12==10.3.9.90
+certifi==2026.6.17
+nvidia-cuda-runtime-cu12==12.8.90
+torchcodec==0.11.1
+toml==0.10.2
+av==15.1.0
+nvidia-cufft-cu12==11.3.3.83
+attrs==26.1.0
+lerobot==0.6.0
+PyYAML==6.0.3
+safetensors==0.8.0
+diffusers==0.35.2
+requests==2.34.2
+nvidia-nvtx-cu12==12.8.90
+charset-normalizer==3.4.9
+networkx==3.6.1
+cmake==4.1.3
+pydantic_core==2.46.4
+timm==1.0.28
+dm-tree==0.1.10
+nvidia-cufile-cu12==1.13.1.3
+termcolor==3.3.0
+multidict==6.7.1
+filelock==3.29.0
+tzdata==2026.3
+aiohappyeyeballs==2.7.1
+httpx==0.28.1
+nvidia-nccl-cu12==2.28.9
+Pygments==2.20.0
+einops==0.8.2
+nvidia-cusparse-cu12==12.5.8.93
+torchvision==0.26.0+cu128
+dill==0.4.1
+GitPython==3.1.51
+nvidia-cuda-cupti-cu12==12.8.90
+decord==0.6.0
+sympy==1.14.0
+cuda-toolkit==12.8.1
+python-dateutil==2.9.0.post0
+shellingham==1.5.4
+propcache==0.5.2
+Farama-Notifications==0.0.6
+pillow==12.2.0
+cuda-bindings==12.9.4
+huggingface_hub==1.23.0
+pandas==2.3.3
+typing-inspect==0.9.0
+packaging==25.0
+rich==15.0.0
+nvidia-cusparselt-cu12==0.7.1
+setuptools==78.1.0
+wrapt==2.2.2
+mypy_extensions==1.1.0
+tqdm==4.68.4
+draccus==0.10.0
+aiosignal==1.4.0
+mpmath==1.3.0
+numpy==2.2.6
+typing_extensions==4.15.0
+psutil==7.2.2
+mdurl==0.1.2
+nvidia-cublas-cu12==12.8.4.1
+absl-py==2.5.0
+gitdb==4.0.12
+transformers==5.5.4
+accelerate==1.14.0
+pydantic==2.13.4
+pip==26.1.2
+markdown-it-py==4.2.0
+typer==0.26.8
+triton==3.6.0
+wandb==0.27.2
+regex==2026.7.10
+yarl==1.24.2
+jsonlines==4.0.0
+mergedeep==1.3.4
+aiohttp==3.14.1
+datasets==4.8.5
+gymnasium==1.3.0
+peft==0.19.1
+zipp==4.1.0
+idna==3.18
+nvidia-cudnn-cu12==9.19.0.56
+h11==0.16.0
+sentry-sdk==2.65.0
+annotated-doc==0.0.4
+protobuf==7.35.1
+typing-inspection==0.4.2
+anyio==4.14.2
+xxhash==3.8.1
+six==1.17.0

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/files/wandb-metadata.json

@@ -0,0 +1,91 @@
+{
+  "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-19T14:04:35.604858Z",
+  "args": [
+    "--config_path=/tmp/insight_lerobot_cfg_8omuu2cw/LAPstyle_RKD_linear_decay_6K.yaml",
+    "--batch_size=128",
+    "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a9df08f7da2d8eb1fa7f28708b4eb6f6dc81b1f9"
+  },
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K",
+  "host": "DGX-H200-01",
+  "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "813929852928"
+    }
+  },
+  "memory": {
+    "total": "2164177309696"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "blcjglxw0aifahoh96yvm0y7dez3b6hi"
+}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_setup.py:_flush():81] Configure stats pid to 2036851
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/logs/debug.log
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K/wandb/run-20260719_230435-bvj1u4hd/logs/debug-internal.log
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:init():766] calling init triggers
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-19 23:04:35,606 INFO    MainThread:2036851 [wandb_init.py:init():814] starting backend
+2026-07-19 23:04:35,830 INFO    MainThread:2036851 [wandb_init.py:init():829] sending inform_init request
+2026-07-19 23:04:36,541 INFO    MainThread:2036851 [wandb_init.py:init():834] backend started and connected
+2026-07-19 23:04:36,546 INFO    MainThread:2036851 [wandb_init.py:init():904] updated telemetry
+2026-07-19 23:04:36,557 INFO    MainThread:2036851 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-19 23:04:37,200 INFO    MainThread:2036851 [wandb_init.py:init():972] starting run threads in backend
+2026-07-19 23:04:37,351 INFO    MainThread:2036851 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-19 23:04:37,353 INFO    MainThread:2036851 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-19 23:04:37,353 INFO    MainThread:2036851 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-19 23:04:37,353 INFO    MainThread:2036851 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-19 23:04:37,357 INFO    MainThread:2036851 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-07-19T23:04:35.943151157+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-19T23:04:35.943433868+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-19T23:04:36.611467609+09:00","level":"INFO","msg":"stream: created new stream","id":"5e3ytmnu"}
+{"time":"2026-07-19T23:04:36.611572242+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-19T23:04:36.611807563+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-19T23:04:36.612175087+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-19T23:04:36.612167211+09:00","level":"INFO","msg":"writer: started","stream_id":"5e3ytmnu"}
+{"time":"2026-07-19T23:04:37.257178205+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-19T23:04:37.743087617+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:04:52.257392987+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":15,"uploaded_len":2}
+{"time":"2026-07-19T23:04:52.542319433+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:07.258163461+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
+{"time":"2026-07-19T23:05:07.619337894+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:22.257738789+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:22.7112944+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:37.257907034+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:37.672011801+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:52.258117309+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:52.582021599+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:07.257961398+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":9,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:07.734131298+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:22.257932664+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:22.547638132+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:37.257258428+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":13,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:37.648933562+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:52.259011675+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":15,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:52.598570588+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:07.257905651+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":17,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:07.578635523+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:22.257818058+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":19,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:22.696487664+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:37.257806194+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":21,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:37.595511834+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:52.257643582+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":23,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:52.60856563+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:07.257778282+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":25,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:07.701328257+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:22.257912985+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":27,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:23.185841344+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:37.258052154+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":29,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:37.808582839+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:52.257994401+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":31,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:52.665256328+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:07.258605414+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":33,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:07.723621911+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:22.257272261+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":35,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:22.635904793+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:37.257219472+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:37.624689221+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:52.257438339+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":39,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:52.665381427+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:07.258034994+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":41,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:07.669561015+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:22.257845366+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":43,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:22.634963442+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:37.258233116+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":45,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:37.684343702+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:52.258036518+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":47,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:52.547934491+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:07.257729046+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":49,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:07.667207125+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:22.257769827+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":51,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:22.626627456+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:37.257737929+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":53,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:37.603320965+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:52.257670755+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":55,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:52.580805954+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:07.257668422+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":57,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:07.681613981+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:22.257992557+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":59,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:22.625503565+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:37.257937592+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":61,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:37.629617001+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:52.258152559+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":63,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:52.578713432+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:07.25791458+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":65,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:07.780341076+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:22.258040527+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":67,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:22.552576795+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:37.25756868+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":69,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:37.548953068+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:52.257907671+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":71,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:52.624452804+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:07.258147285+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":73,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:07.738971805+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:22.257660414+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":75,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:22.743119329+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:37.258115457+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":77,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:37.561528774+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-07-19 23:04:35,706 INFO    MainThread:2036849 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-07-19 23:04:35,706 INFO    MainThread:2036849 [wandb_setup.py:_flush():81] Configure stats pid to 2036849
+2026-07-19 23:04:35,706 INFO    MainThread:2036849 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-19 23:04:35,706 INFO    MainThread:2036849 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/logs/debug.log
+2026-07-19 23:04:35,706 INFO    MainThread:2036849 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/logs/debug-internal.log
+2026-07-19 23:04:35,706 INFO    MainThread:2036849 [wandb_init.py:init():766] calling init triggers
+2026-07-19 23:04:35,706 INFO    MainThread:2036849 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
+2026-07-19 23:04:35,707 INFO    MainThread:2036849 [wandb_init.py:init():814] starting backend
+2026-07-19 23:04:35,932 INFO    MainThread:2036849 [wandb_init.py:init():829] sending inform_init request
+2026-07-19 23:04:36,613 INFO    MainThread:2036849 [wandb_init.py:init():834] backend started and connected
+2026-07-19 23:04:36,615 INFO    MainThread:2036849 [wandb_init.py:init():904] updated telemetry
+2026-07-19 23:04:36,623 INFO    MainThread:2036849 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-07-19 23:04:37,083 INFO    MainThread:2036849 [wandb_init.py:init():972] starting run threads in backend
+2026-07-19 23:04:37,250 INFO    MainThread:2036849 [wandb_run.py:_console_start():2523] atexit reg
+2026-07-19 23:04:37,250 INFO    MainThread:2036849 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-07-19 23:04:37,250 INFO    MainThread:2036849 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-07-19 23:04:37,251 INFO    MainThread:2036849 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-07-19 23:04:37,254 INFO    MainThread:2036849 [wandb_init.py:init():1010] run started, returning control to user process

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/files/output.log

@@ -0,0 +1,17 @@
+INFO 2026-07-19 23:04:37 db_utils.py:121 Logs will be synced with wandb.
+INFO 2026-07-19 23:04:37 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/5e3ytmnu
+INFO 2026-07-19 23:04:37 ot_train.py:298 Creating dataset
+INFO 2026-07-19 23:04:39 ot_train.py:332 Creating policy
+Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 1496.60it/s]
+`torch_dtype` is deprecated! Use `dtype` instead!
+Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1886.97it/s]
+INFO 2026-07-19 23:04:45 ot_train.py:405 Creating optimizer and scheduler
+INFO 2026-07-19 23:04:45 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K
+INFO 2026-07-19 23:04:45 ot_train.py:446 cfg.steps=60000 (60K)
+INFO 2026-07-19 23:04:45 ot_train.py:447 dataset.num_frames=218367 (218K)
+INFO 2026-07-19 23:04:45 ot_train.py:448 dataset.num_episodes=4930
+INFO 2026-07-19 23:04:45 ot_train.py:451 Effective batch size: 128 x 1 = 128
+INFO 2026-07-19 23:04:45 ot_train.py:452 num_learnable_params=1622090881 (2B)
+INFO 2026-07-19 23:04:45 ot_train.py:453 num_total_params=3145712033 (3B)
+Training:   0%|          | 0/60000 [00:00<?, ?step/s]INFO 2026-07-19 23:04:45 ot_train.py:604 Start offline training on a fixed dataset, with effective batch size: 128
+Training:   0%|          | 182/60000 [09:50<51:30:35,  3.10s/step]

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/files/requirements.txt

@@ -0,0 +1,117 @@
+pyarrow==25.0.0
+importlib_metadata==9.0.0
+wheel==0.47.0
+cloudpickle==3.1.2
+nvidia-cusolver-cu12==11.7.3.90
+multiprocess==0.70.19
+torch==2.11.0+cu128
+nvidia-nvjitlink-cu12==12.8.93
+fsspec==2026.2.0
+pytz==2026.2
+hf-xet==1.5.1
+tokenizers==0.22.2
+opencv-python-headless==4.13.0.92
+cuda-pathfinder==1.2.2
+frozenlist==1.8.0
+smmap==5.0.3
+httpcore==1.0.9
+nvidia-nvshmem-cu12==3.4.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+annotated-types==0.7.0
+platformdirs==4.10.0
+click==8.4.2
+pyyaml-include==1.4.1
+MarkupSafe==3.0.3
+Jinja2==3.1.6
+urllib3==2.7.0
+nvidia-curand-cu12==10.3.9.90
+certifi==2026.6.17
+nvidia-cuda-runtime-cu12==12.8.90
+torchcodec==0.11.1
+toml==0.10.2
+av==15.1.0
+nvidia-cufft-cu12==11.3.3.83
+attrs==26.1.0
+lerobot==0.6.0
+PyYAML==6.0.3
+safetensors==0.8.0
+diffusers==0.35.2
+requests==2.34.2
+nvidia-nvtx-cu12==12.8.90
+charset-normalizer==3.4.9
+networkx==3.6.1
+cmake==4.1.3
+pydantic_core==2.46.4
+timm==1.0.28
+dm-tree==0.1.10
+nvidia-cufile-cu12==1.13.1.3
+termcolor==3.3.0
+multidict==6.7.1
+filelock==3.29.0
+tzdata==2026.3
+aiohappyeyeballs==2.7.1
+httpx==0.28.1
+nvidia-nccl-cu12==2.28.9
+Pygments==2.20.0
+einops==0.8.2
+nvidia-cusparse-cu12==12.5.8.93
+torchvision==0.26.0+cu128
+dill==0.4.1
+GitPython==3.1.51
+nvidia-cuda-cupti-cu12==12.8.90
+decord==0.6.0
+sympy==1.14.0
+cuda-toolkit==12.8.1
+python-dateutil==2.9.0.post0
+shellingham==1.5.4
+propcache==0.5.2
+Farama-Notifications==0.0.6
+pillow==12.2.0
+cuda-bindings==12.9.4
+huggingface_hub==1.23.0
+pandas==2.3.3
+typing-inspect==0.9.0
+packaging==25.0
+rich==15.0.0
+nvidia-cusparselt-cu12==0.7.1
+setuptools==78.1.0
+wrapt==2.2.2
+mypy_extensions==1.1.0
+tqdm==4.68.4
+draccus==0.10.0
+aiosignal==1.4.0
+mpmath==1.3.0
+numpy==2.2.6
+typing_extensions==4.15.0
+psutil==7.2.2
+mdurl==0.1.2
+nvidia-cublas-cu12==12.8.4.1
+absl-py==2.5.0
+gitdb==4.0.12
+transformers==5.5.4
+accelerate==1.14.0
+pydantic==2.13.4
+pip==26.1.2
+markdown-it-py==4.2.0
+typer==0.26.8
+triton==3.6.0
+wandb==0.27.2
+regex==2026.7.10
+yarl==1.24.2
+jsonlines==4.0.0
+mergedeep==1.3.4
+aiohttp==3.14.1
+datasets==4.8.5
+gymnasium==1.3.0
+peft==0.19.1
+zipp==4.1.0
+idna==3.18
+nvidia-cudnn-cu12==9.19.0.56
+h11==0.16.0
+sentry-sdk==2.65.0
+annotated-doc==0.0.4
+protobuf==7.35.1
+typing-inspection==0.4.2
+anyio==4.14.2
+xxhash==3.8.1
+six==1.17.0

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/files/wandb-metadata.json

@@ -0,0 +1,91 @@
+{
+  "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-07-19T14:04:35.704983Z",
+  "args": [
+    "--config_path=/tmp/insight_lerobot_cfg_7rsw3qpw/RSCLstyle_RKD_cosine_decay_60K.yaml",
+    "--batch_size=128",
+    "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K"
+  ],
+  "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
+  "git": {
+    "remote": "git@github.com:paragon7060/clvla.git",
+    "commit": "a9df08f7da2d8eb1fa7f28708b4eb6f6dc81b1f9"
+  },
+  "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K",
+  "host": "DGX-H200-01",
+  "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888425144320",
+      "used": "813929852928"
+    }
+  },
+  "memory": {
+    "total": "2164177309696"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f"
+    },
+    {
+      "name": "NVIDIA H200",
+      "memoryTotal": "150754820096",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c"
+    }
+  ],
+  "cudaVersion": "12.9",
+  "writerId": "gv4s93wd7dns6rviy8dm02jp22skkffq"
+}

Abs_6D + DeltaEE/_archive/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/logs/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-07-19T23:04:35.943151157+09:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-19T23:04:35.943433868+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-07-19T23:04:36.611467609+09:00","level":"INFO","msg":"stream: created new stream","id":"5e3ytmnu"}
+{"time":"2026-07-19T23:04:36.611572242+09:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-19T23:04:36.611807563+09:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-19T23:04:36.612175087+09:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-19T23:04:36.612167211+09:00","level":"INFO","msg":"writer: started","stream_id":"5e3ytmnu"}
+{"time":"2026-07-19T23:04:37.257178205+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-19T23:04:37.743087617+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:04:52.257392987+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":15,"uploaded_len":2}
+{"time":"2026-07-19T23:04:52.542319433+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:07.258163461+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
+{"time":"2026-07-19T23:05:07.619337894+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:22.257738789+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:22.7112944+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:37.257907034+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:37.672011801+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:05:52.258117309+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:05:52.582021599+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:07.257961398+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":9,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:07.734131298+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:22.257932664+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:22.547638132+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:37.257258428+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":13,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:37.648933562+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:06:52.259011675+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":15,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:06:52.598570588+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:07.257905651+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":17,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:07.578635523+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:22.257818058+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":19,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:22.696487664+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:37.257806194+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":21,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:37.595511834+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:07:52.257643582+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":23,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:07:52.60856563+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:07.257778282+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":25,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:07.701328257+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:22.257912985+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":27,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:23.185841344+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:37.258052154+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":29,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:37.808582839+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:08:52.257994401+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":31,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:08:52.665256328+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:07.258605414+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":33,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:07.723621911+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:22.257272261+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":35,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:22.635904793+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:37.257219472+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:37.624689221+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:09:52.257438339+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":39,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:09:52.665381427+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:07.258034994+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":41,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:07.669561015+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:22.257845366+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":43,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:22.634963442+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:37.258233116+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":45,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:37.684343702+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:10:52.258036518+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":47,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:10:52.547934491+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:07.257729046+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":49,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:07.667207125+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:22.257769827+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":51,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:22.626627456+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:37.257737929+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":53,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:37.603320965+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:11:52.257670755+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":55,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:11:52.580805954+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:07.257668422+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":57,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:07.681613981+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:22.257992557+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":59,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:22.625503565+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:37.257937592+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":61,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:37.629617001+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:12:52.258152559+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":63,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:12:52.578713432+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:07.25791458+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":65,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:07.780341076+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:22.258040527+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":67,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:22.552576795+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:37.25756868+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":69,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:37.548953068+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:13:52.257907671+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":71,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:13:52.624452804+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:07.258147285+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":73,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:07.738971805+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:22.257660414+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":75,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:22.743119329+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-19T23:14:37.258115457+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":77,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-07-19T23:14:37.561528774+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}