Whalswp commited on
Commit
aa55d05
·
verified ·
1 Parent(s): 002a367

Add files using upload-large-folder tool

Browse files
Files changed (34) hide show
  1. .gitattributes +1 -0
  2. Abs_Joint/Baseline/action_space_manifest.json +124 -0
  3. Abs_Joint/Baseline/checkpoints/060000/action_space_manifest.json +124 -0
  4. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/action_space_manifest.json +124 -0
  5. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/config.json +107 -0
  6. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/model.safetensors +3 -0
  7. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor.json +23 -0
  8. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors +3 -0
  9. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor.json +78 -0
  10. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors +3 -0
  11. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/prompt_manifest.json +40 -0
  12. Abs_Joint/Baseline/checkpoints/060000/pretrained_model/train_config.json +258 -0
  13. Abs_Joint/Baseline/checkpoints/060000/prompt_manifest.json +40 -0
  14. Abs_Joint/Baseline/checkpoints/060000/resolved_config.yaml +24 -0
  15. Abs_Joint/Baseline/checkpoints/060000/source_config.yaml +63 -0
  16. Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_param_groups.json +577 -0
  17. Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_state.safetensors +3 -0
  18. Abs_Joint/Baseline/checkpoints/060000/training_state/rng_state.safetensors +3 -0
  19. Abs_Joint/Baseline/checkpoints/060000/training_state/scheduler_state.json +18 -0
  20. Abs_Joint/Baseline/checkpoints/060000/training_state/training_step.json +5 -0
  21. Abs_Joint/Baseline/prompt_manifest.json +40 -0
  22. Abs_Joint/Baseline/resolved_config.yaml +24 -0
  23. Abs_Joint/Baseline/source_config.yaml +63 -0
  24. Abs_Joint/Baseline/wandb/debug-internal.log +0 -0
  25. Abs_Joint/Baseline/wandb/debug.log +45 -0
  26. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/config.yaml +300 -0
  27. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/output.log +320 -0
  28. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/requirements.txt +117 -0
  29. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-metadata.json +64 -0
  30. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-summary.json +1 -0
  31. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-core.log +16 -0
  32. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log +0 -0
  33. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log +45 -0
  34. Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/run-a3ogbmg4.wandb +3 -0
.gitattributes CHANGED
@@ -43,3 +43,4 @@ Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/run-1jmjb68j.wandb filter=lfs
43
  Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/run-pebh9ode.wandb filter=lfs diff=lfs merge=lfs -text
44
  Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_10K/wandb/run-20260715_011448-fbqzmp5m/run-fbqzmp5m.wandb filter=lfs diff=lfs merge=lfs -text
45
  Abs_6D/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260715_102129-typqzcxc/run-typqzcxc.wandb filter=lfs diff=lfs merge=lfs -text
 
 
43
  Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/run-pebh9ode.wandb filter=lfs diff=lfs merge=lfs -text
44
  Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_10K/wandb/run-20260715_011448-fbqzmp5m/run-fbqzmp5m.wandb filter=lfs diff=lfs merge=lfs -text
45
  Abs_6D/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260715_102129-typqzcxc/run-typqzcxc.wandb filter=lfs diff=lfs merge=lfs -text
46
+ Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/run-a3ogbmg4.wandb filter=lfs diff=lfs merge=lfs -text
Abs_Joint/Baseline/action_space_manifest.json ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 2,
3
+ "action_space": "joint_abs",
4
+ "action_views": [
5
+ "joint_abs"
6
+ ],
7
+ "source_feature": "action.joint_abs",
8
+ "canonical_feature": "action",
9
+ "dtype": "float32",
10
+ "shape": [
11
+ 8
12
+ ],
13
+ "names": [
14
+ "j1",
15
+ "j2",
16
+ "j3",
17
+ "j4",
18
+ "j5",
19
+ "j6",
20
+ "j7",
21
+ "gripper"
22
+ ],
23
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
24
+ "dataset_revision": "v3.0",
25
+ "dataset_total_frames": 218367,
26
+ "dataset_total_episodes": 4930,
27
+ "dataset_fps": 10,
28
+ "stats_sha256": "4a35bfed6a90ddd1401509423c3b13ffc9adbcc4ffe461e1be341859505c2cc3",
29
+ "stats": {
30
+ "min": [
31
+ -2.8872478008270264,
32
+ -1.7627999782562256,
33
+ -2.8033182621002197,
34
+ -3.0717999935150146,
35
+ -2.887302875518799,
36
+ -0.012209548614919186,
37
+ -2.9217312335968018,
38
+ 0.0
39
+ ],
40
+ "max": [
41
+ 2.880279302597046,
42
+ 1.5605382919311523,
43
+ 2.884284496307373,
44
+ -0.7739977240562439,
45
+ 2.8875293731689453,
46
+ 3.7483208179473877,
47
+ 2.9222261905670166,
48
+ 0.03999999910593033
49
+ ],
50
+ "mean": [
51
+ 0.0019617623927398265,
52
+ -0.6096056340962691,
53
+ -0.08146021469498298,
54
+ -1.9318156007993963,
55
+ 0.13481075339339976,
56
+ 2.1488292862501455,
57
+ 0.20664679119827345,
58
+ 0.021167300266524632
59
+ ],
60
+ "std": [
61
+ 0.4184733225263563,
62
+ 0.43452186167386714,
63
+ 0.34657299197762675,
64
+ 0.31065299277285635,
65
+ 0.5804164362256183,
66
+ 0.560995779575591,
67
+ 1.4846336752340794,
68
+ 0.01996590431050817
69
+ ],
70
+ "count": [
71
+ 218367
72
+ ],
73
+ "q01": [
74
+ -0.21136459407246144,
75
+ -1.1820493696591017,
76
+ -0.23617874574281436,
77
+ -2.3513137651342415,
78
+ -0.13724740781131373,
79
+ 1.8803846119895578,
80
+ -1.1764070030629525,
81
+ -1.000000013351432e-10
82
+ ],
83
+ "q10": [
84
+ -0.17717514597274497,
85
+ -0.9819092294296998,
86
+ -0.21149320673037963,
87
+ -2.1958533677694447,
88
+ -0.0927211042478635,
89
+ 1.9384504318973095,
90
+ -0.9679293888972026,
91
+ -1.000000013351432e-10
92
+ ],
93
+ "q50": [
94
+ 0.004311903393904223,
95
+ -0.5611143444409522,
96
+ -0.09504186980105549,
97
+ -1.8999666287578785,
98
+ 0.155166053430988,
99
+ 2.113795242440475,
100
+ 0.2503060374532495,
101
+ 0.020772594934870346
102
+ ],
103
+ "q90": [
104
+ 0.16607161790312464,
105
+ -0.4243375581079769,
106
+ 0.08872114509119819,
107
+ -1.7717546326449354,
108
+ 0.302495674015553,
109
+ 2.4791201268598524,
110
+ 1.2829547587923422,
111
+ 0.039998362213352394
112
+ ],
113
+ "q99": [
114
+ 0.19894254444240483,
115
+ -0.3908672692345395,
116
+ 0.13510704087246417,
117
+ -1.7149038786707966,
118
+ 0.33183211224695647,
119
+ 2.6996600881852637,
120
+ 1.4893457903653573,
121
+ 0.03999983541667277
122
+ ]
123
+ }
124
+ }
Abs_Joint/Baseline/checkpoints/060000/action_space_manifest.json ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 2,
3
+ "action_space": "joint_abs",
4
+ "action_views": [
5
+ "joint_abs"
6
+ ],
7
+ "source_feature": "action.joint_abs",
8
+ "canonical_feature": "action",
9
+ "dtype": "float32",
10
+ "shape": [
11
+ 8
12
+ ],
13
+ "names": [
14
+ "j1",
15
+ "j2",
16
+ "j3",
17
+ "j4",
18
+ "j5",
19
+ "j6",
20
+ "j7",
21
+ "gripper"
22
+ ],
23
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
24
+ "dataset_revision": "v3.0",
25
+ "dataset_total_frames": 218367,
26
+ "dataset_total_episodes": 4930,
27
+ "dataset_fps": 10,
28
+ "stats_sha256": "4a35bfed6a90ddd1401509423c3b13ffc9adbcc4ffe461e1be341859505c2cc3",
29
+ "stats": {
30
+ "min": [
31
+ -2.8872478008270264,
32
+ -1.7627999782562256,
33
+ -2.8033182621002197,
34
+ -3.0717999935150146,
35
+ -2.887302875518799,
36
+ -0.012209548614919186,
37
+ -2.9217312335968018,
38
+ 0.0
39
+ ],
40
+ "max": [
41
+ 2.880279302597046,
42
+ 1.5605382919311523,
43
+ 2.884284496307373,
44
+ -0.7739977240562439,
45
+ 2.8875293731689453,
46
+ 3.7483208179473877,
47
+ 2.9222261905670166,
48
+ 0.03999999910593033
49
+ ],
50
+ "mean": [
51
+ 0.0019617623927398265,
52
+ -0.6096056340962691,
53
+ -0.08146021469498298,
54
+ -1.9318156007993963,
55
+ 0.13481075339339976,
56
+ 2.1488292862501455,
57
+ 0.20664679119827345,
58
+ 0.021167300266524632
59
+ ],
60
+ "std": [
61
+ 0.4184733225263563,
62
+ 0.43452186167386714,
63
+ 0.34657299197762675,
64
+ 0.31065299277285635,
65
+ 0.5804164362256183,
66
+ 0.560995779575591,
67
+ 1.4846336752340794,
68
+ 0.01996590431050817
69
+ ],
70
+ "count": [
71
+ 218367
72
+ ],
73
+ "q01": [
74
+ -0.21136459407246144,
75
+ -1.1820493696591017,
76
+ -0.23617874574281436,
77
+ -2.3513137651342415,
78
+ -0.13724740781131373,
79
+ 1.8803846119895578,
80
+ -1.1764070030629525,
81
+ -1.000000013351432e-10
82
+ ],
83
+ "q10": [
84
+ -0.17717514597274497,
85
+ -0.9819092294296998,
86
+ -0.21149320673037963,
87
+ -2.1958533677694447,
88
+ -0.0927211042478635,
89
+ 1.9384504318973095,
90
+ -0.9679293888972026,
91
+ -1.000000013351432e-10
92
+ ],
93
+ "q50": [
94
+ 0.004311903393904223,
95
+ -0.5611143444409522,
96
+ -0.09504186980105549,
97
+ -1.8999666287578785,
98
+ 0.155166053430988,
99
+ 2.113795242440475,
100
+ 0.2503060374532495,
101
+ 0.020772594934870346
102
+ ],
103
+ "q90": [
104
+ 0.16607161790312464,
105
+ -0.4243375581079769,
106
+ 0.08872114509119819,
107
+ -1.7717546326449354,
108
+ 0.302495674015553,
109
+ 2.4791201268598524,
110
+ 1.2829547587923422,
111
+ 0.039998362213352394
112
+ ],
113
+ "q99": [
114
+ 0.19894254444240483,
115
+ -0.3908672692345395,
116
+ 0.13510704087246417,
117
+ -1.7149038786707966,
118
+ 0.33183211224695647,
119
+ 2.6996600881852637,
120
+ 1.4893457903653573,
121
+ 0.03999983541667277
122
+ ]
123
+ }
124
+ }
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/action_space_manifest.json ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 2,
3
+ "action_space": "joint_abs",
4
+ "action_views": [
5
+ "joint_abs"
6
+ ],
7
+ "source_feature": "action.joint_abs",
8
+ "canonical_feature": "action",
9
+ "dtype": "float32",
10
+ "shape": [
11
+ 8
12
+ ],
13
+ "names": [
14
+ "j1",
15
+ "j2",
16
+ "j3",
17
+ "j4",
18
+ "j5",
19
+ "j6",
20
+ "j7",
21
+ "gripper"
22
+ ],
23
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
24
+ "dataset_revision": "v3.0",
25
+ "dataset_total_frames": 218367,
26
+ "dataset_total_episodes": 4930,
27
+ "dataset_fps": 10,
28
+ "stats_sha256": "4a35bfed6a90ddd1401509423c3b13ffc9adbcc4ffe461e1be341859505c2cc3",
29
+ "stats": {
30
+ "min": [
31
+ -2.8872478008270264,
32
+ -1.7627999782562256,
33
+ -2.8033182621002197,
34
+ -3.0717999935150146,
35
+ -2.887302875518799,
36
+ -0.012209548614919186,
37
+ -2.9217312335968018,
38
+ 0.0
39
+ ],
40
+ "max": [
41
+ 2.880279302597046,
42
+ 1.5605382919311523,
43
+ 2.884284496307373,
44
+ -0.7739977240562439,
45
+ 2.8875293731689453,
46
+ 3.7483208179473877,
47
+ 2.9222261905670166,
48
+ 0.03999999910593033
49
+ ],
50
+ "mean": [
51
+ 0.0019617623927398265,
52
+ -0.6096056340962691,
53
+ -0.08146021469498298,
54
+ -1.9318156007993963,
55
+ 0.13481075339339976,
56
+ 2.1488292862501455,
57
+ 0.20664679119827345,
58
+ 0.021167300266524632
59
+ ],
60
+ "std": [
61
+ 0.4184733225263563,
62
+ 0.43452186167386714,
63
+ 0.34657299197762675,
64
+ 0.31065299277285635,
65
+ 0.5804164362256183,
66
+ 0.560995779575591,
67
+ 1.4846336752340794,
68
+ 0.01996590431050817
69
+ ],
70
+ "count": [
71
+ 218367
72
+ ],
73
+ "q01": [
74
+ -0.21136459407246144,
75
+ -1.1820493696591017,
76
+ -0.23617874574281436,
77
+ -2.3513137651342415,
78
+ -0.13724740781131373,
79
+ 1.8803846119895578,
80
+ -1.1764070030629525,
81
+ -1.000000013351432e-10
82
+ ],
83
+ "q10": [
84
+ -0.17717514597274497,
85
+ -0.9819092294296998,
86
+ -0.21149320673037963,
87
+ -2.1958533677694447,
88
+ -0.0927211042478635,
89
+ 1.9384504318973095,
90
+ -0.9679293888972026,
91
+ -1.000000013351432e-10
92
+ ],
93
+ "q50": [
94
+ 0.004311903393904223,
95
+ -0.5611143444409522,
96
+ -0.09504186980105549,
97
+ -1.8999666287578785,
98
+ 0.155166053430988,
99
+ 2.113795242440475,
100
+ 0.2503060374532495,
101
+ 0.020772594934870346
102
+ ],
103
+ "q90": [
104
+ 0.16607161790312464,
105
+ -0.4243375581079769,
106
+ 0.08872114509119819,
107
+ -1.7717546326449354,
108
+ 0.302495674015553,
109
+ 2.4791201268598524,
110
+ 1.2829547587923422,
111
+ 0.039998362213352394
112
+ ],
113
+ "q99": [
114
+ 0.19894254444240483,
115
+ -0.3908672692345395,
116
+ 0.13510704087246417,
117
+ -1.7149038786707966,
118
+ 0.33183211224695647,
119
+ 2.6996600881852637,
120
+ 1.4893457903653573,
121
+ 0.03999983541667277
122
+ ]
123
+ }
124
+ }
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/config.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "groot",
3
+ "n_obs_steps": 1,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 16
9
+ ]
10
+ },
11
+ "observation.images.wrist": {
12
+ "type": "VISUAL",
13
+ "shape": [
14
+ 3,
15
+ 224,
16
+ 224
17
+ ]
18
+ },
19
+ "observation.images.right_shoulder": {
20
+ "type": "VISUAL",
21
+ "shape": [
22
+ 3,
23
+ 224,
24
+ 224
25
+ ]
26
+ },
27
+ "observation.images.guide": {
28
+ "type": "VISUAL",
29
+ "shape": [
30
+ 3,
31
+ 224,
32
+ 224
33
+ ]
34
+ }
35
+ },
36
+ "output_features": {
37
+ "action": {
38
+ "type": "ACTION",
39
+ "shape": [
40
+ 8
41
+ ]
42
+ }
43
+ },
44
+ "device": "cuda",
45
+ "use_amp": false,
46
+ "use_peft": false,
47
+ "push_to_hub": false,
48
+ "repo_id": null,
49
+ "private": null,
50
+ "tags": null,
51
+ "license": null,
52
+ "pretrained_path": null,
53
+ "pretrained_revision": null,
54
+ "chunk_size": 16,
55
+ "n_action_steps": 16,
56
+ "max_state_dim": 132,
57
+ "max_action_dim": 132,
58
+ "normalization_mapping": {
59
+ "VISUAL": "IDENTITY",
60
+ "STATE": "IDENTITY",
61
+ "ACTION": "IDENTITY"
62
+ },
63
+ "base_model_path": "nvidia/GR00T-N1.7-3B",
64
+ "action_decode_transform": null,
65
+ "embodiment_tag": "new_embodiment",
66
+ "tune_llm": false,
67
+ "tune_visual": false,
68
+ "tune_projector": true,
69
+ "tune_diffusion_model": true,
70
+ "tune_vlln": true,
71
+ "tune_top_llm_layers": 0,
72
+ "num_inference_timesteps": null,
73
+ "rtc_ramp_rate": null,
74
+ "use_flash_attention": false,
75
+ "use_relative_actions": false,
76
+ "relative_exclude_joints": [],
77
+ "optimizer_lr": 0.0001,
78
+ "optimizer_betas": [
79
+ 0.9,
80
+ 0.999
81
+ ],
82
+ "optimizer_eps": 1e-08,
83
+ "optimizer_weight_decay": 1e-05,
84
+ "warmup_ratio": 0.05,
85
+ "use_bf16": true,
86
+ "model_params_fp32": true,
87
+ "image_size": [
88
+ 256,
89
+ 256
90
+ ],
91
+ "tokenizer_assets_repo": null,
92
+ "lora_rank": 0,
93
+ "lora_alpha": 16,
94
+ "lora_dropout": 0.1,
95
+ "lora_full_model": false,
96
+ "video_backend": "decord",
97
+ "balance_dataset_weights": true,
98
+ "balance_trajectory_weights": true,
99
+ "dataset_paths": null,
100
+ "output_dir": "./tmp/gr00t",
101
+ "save_steps": 1000,
102
+ "max_steps": 10000,
103
+ "batch_size": 32,
104
+ "dataloader_num_workers": 8,
105
+ "report_to": "wandb",
106
+ "resume": false
107
+ }
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf3e0351d7c4f730a00c9e511dc72d7be818363889f68430b3251e8ad1a99747
3
+ size 12576215296
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "groot_action_unpack_unnormalize_v2",
6
+ "config": {
7
+ "env_action_dim": 8,
8
+ "normalize_min_max": true,
9
+ "clip_normalized_action": true,
10
+ "libero_gripper_action": false,
11
+ "libero_gripper_binarize": true
12
+ },
13
+ "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
14
+ },
15
+ {
16
+ "registry_name": "device_processor",
17
+ "config": {
18
+ "device": "cpu",
19
+ "float_dtype": null
20
+ }
21
+ }
22
+ ]
23
+ }
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f21dbfaa54b712cd34c86e2f401159550987ceb71c54934d3e7fa600b69b15f8
3
+ size 5608
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor.json ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {}
8
+ }
9
+ },
10
+ {
11
+ "registry_name": "to_batch_processor",
12
+ "config": {}
13
+ },
14
+ {
15
+ "registry_name": "groot_n1_7_pack_inputs_v1",
16
+ "config": {
17
+ "state_horizon": 1,
18
+ "action_horizon": 16,
19
+ "valid_action_horizon": 16,
20
+ "video_horizon": null,
21
+ "max_state_dim": 132,
22
+ "max_action_dim": 132,
23
+ "language_key": "task",
24
+ "formalize_language": true,
25
+ "embodiment_tag": "new_embodiment",
26
+ "embodiment_mapping": {
27
+ "oxe_droid_relative_eef_relative_joint": 24,
28
+ "xdof_relative_eef_relative_joint": 27,
29
+ "xdof_relative_eef_relative_joint_subtask": 27,
30
+ "real_g1_relative_eef_relative_joints": 25,
31
+ "real_r1_pro_sharpa_relative_eef": 26,
32
+ "real_r1_pro_sharpa_relative_eef_human": 26,
33
+ "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
34
+ "real_r1_pro_sharpa_relative_eef_mecka": 26,
35
+ "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
36
+ "simpler_env_google": 0,
37
+ "simpler_env_widowx": 1,
38
+ "libero_sim": 2,
39
+ "new_embodiment": 10
40
+ },
41
+ "normalize_min_max": true,
42
+ "state_dropout_prob": 0.0,
43
+ "clip_outliers": true,
44
+ "use_percentiles": false,
45
+ "video_modality_keys": null,
46
+ "raw_stats": null,
47
+ "modality_config": null
48
+ },
49
+ "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors"
50
+ },
51
+ {
52
+ "registry_name": "groot_n1_7_vlm_encode_v1",
53
+ "config": {
54
+ "model_name": "nvidia/Cosmos-Reason2-2B",
55
+ "image_crop_size": [
56
+ 230,
57
+ 230
58
+ ],
59
+ "image_target_size": [
60
+ 256,
61
+ 256
62
+ ],
63
+ "shortest_image_edge": null,
64
+ "crop_fraction": null,
65
+ "use_albumentations": false,
66
+ "letter_box_transform": false,
67
+ "device": "cuda"
68
+ }
69
+ },
70
+ {
71
+ "registry_name": "device_processor",
72
+ "config": {
73
+ "device": "cuda",
74
+ "float_dtype": null
75
+ }
76
+ }
77
+ ]
78
+ }
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f21dbfaa54b712cd34c86e2f401159550987ceb71c54934d3e7fa600b69b15f8
3
+ size 5608
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/prompt_manifest.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "prompt_set": "detailed",
4
+ "source_file": "task_prompts.json",
5
+ "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
6
+ "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
7
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
8
+ "dataset_revision": "v3.0",
9
+ "task_count": 13,
10
+ "task_codes": [
11
+ "5f",
12
+ "5g",
13
+ "1ext",
14
+ "3b",
15
+ "5b",
16
+ "3a",
17
+ "3c",
18
+ "5h",
19
+ "5d",
20
+ "3d",
21
+ "5e",
22
+ "5a",
23
+ "5c"
24
+ ],
25
+ "prompts": {
26
+ "5f": "Close the bottle in clockwise direction.",
27
+ "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
28
+ "1ext": "Find the arrow guide and open the indicated drawer.",
29
+ "3b": "Open the door, rotate counter-clockwise and push.",
30
+ "5b": "Open the bottle in counter-clockwise direction.",
31
+ "3a": "Open the door, rotate clockwise and push.",
32
+ "3c": "Open the door, rotate clockwise and pull.",
33
+ "5h": "Close the bottle in counter-clockwise direction.",
34
+ "5d": "Close the bottle in clockwise direction.",
35
+ "3d": "Open the door, rotate counter-clockwise and pull.",
36
+ "5e": "Close the bottle in counter-clockwise direction.",
37
+ "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
38
+ "5c": "Open the bottle in clockwise direction."
39
+ }
40
+ }
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/train_config.json ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
4
+ "root": "/home/seonho/INSIGHTfixposV4_filtered_multispace_v2",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "tfs": {
11
+ "brightness": {
12
+ "weight": 1.0,
13
+ "type": "ColorJitter",
14
+ "kwargs": {
15
+ "brightness": [
16
+ 0.8,
17
+ 1.2
18
+ ]
19
+ }
20
+ },
21
+ "contrast": {
22
+ "weight": 1.0,
23
+ "type": "ColorJitter",
24
+ "kwargs": {
25
+ "contrast": [
26
+ 0.8,
27
+ 1.2
28
+ ]
29
+ }
30
+ },
31
+ "saturation": {
32
+ "weight": 1.0,
33
+ "type": "ColorJitter",
34
+ "kwargs": {
35
+ "saturation": [
36
+ 0.5,
37
+ 1.5
38
+ ]
39
+ }
40
+ },
41
+ "hue": {
42
+ "weight": 1.0,
43
+ "type": "ColorJitter",
44
+ "kwargs": {
45
+ "hue": [
46
+ -0.05,
47
+ 0.05
48
+ ]
49
+ }
50
+ },
51
+ "sharpness": {
52
+ "weight": 1.0,
53
+ "type": "SharpnessJitter",
54
+ "kwargs": {
55
+ "sharpness": [
56
+ 0.5,
57
+ 1.5
58
+ ]
59
+ }
60
+ },
61
+ "affine": {
62
+ "weight": 1.0,
63
+ "type": "RandomAffine",
64
+ "kwargs": {
65
+ "degrees": [
66
+ -5.0,
67
+ 5.0
68
+ ],
69
+ "translate": [
70
+ 0.05,
71
+ 0.05
72
+ ]
73
+ }
74
+ }
75
+ }
76
+ },
77
+ "revision": null,
78
+ "use_imagenet_stats": true,
79
+ "video_backend": "torchcodec",
80
+ "return_uint8": false,
81
+ "depth_output_unit": "mm",
82
+ "streaming": false,
83
+ "eval_split": 0.0
84
+ },
85
+ "env": null,
86
+ "policy": {
87
+ "type": "groot",
88
+ "n_obs_steps": 1,
89
+ "input_features": {
90
+ "observation.state": {
91
+ "type": "STATE",
92
+ "shape": [
93
+ 16
94
+ ]
95
+ },
96
+ "observation.images.wrist": {
97
+ "type": "VISUAL",
98
+ "shape": [
99
+ 3,
100
+ 224,
101
+ 224
102
+ ]
103
+ },
104
+ "observation.images.right_shoulder": {
105
+ "type": "VISUAL",
106
+ "shape": [
107
+ 3,
108
+ 224,
109
+ 224
110
+ ]
111
+ },
112
+ "observation.images.guide": {
113
+ "type": "VISUAL",
114
+ "shape": [
115
+ 3,
116
+ 224,
117
+ 224
118
+ ]
119
+ }
120
+ },
121
+ "output_features": {
122
+ "action": {
123
+ "type": "ACTION",
124
+ "shape": [
125
+ 8
126
+ ]
127
+ }
128
+ },
129
+ "device": "cuda",
130
+ "use_amp": false,
131
+ "use_peft": false,
132
+ "push_to_hub": false,
133
+ "repo_id": null,
134
+ "private": null,
135
+ "tags": null,
136
+ "license": null,
137
+ "pretrained_path": null,
138
+ "pretrained_revision": null,
139
+ "chunk_size": 16,
140
+ "n_action_steps": 16,
141
+ "max_state_dim": 132,
142
+ "max_action_dim": 132,
143
+ "normalization_mapping": {
144
+ "VISUAL": "IDENTITY",
145
+ "STATE": "IDENTITY",
146
+ "ACTION": "IDENTITY"
147
+ },
148
+ "base_model_path": "nvidia/GR00T-N1.7-3B",
149
+ "action_decode_transform": null,
150
+ "embodiment_tag": "new_embodiment",
151
+ "tune_llm": false,
152
+ "tune_visual": false,
153
+ "tune_projector": true,
154
+ "tune_diffusion_model": true,
155
+ "tune_vlln": true,
156
+ "tune_top_llm_layers": 0,
157
+ "num_inference_timesteps": null,
158
+ "rtc_ramp_rate": null,
159
+ "use_flash_attention": false,
160
+ "use_relative_actions": false,
161
+ "relative_exclude_joints": [],
162
+ "optimizer_lr": 0.0001,
163
+ "optimizer_betas": [
164
+ 0.9,
165
+ 0.999
166
+ ],
167
+ "optimizer_eps": 1e-08,
168
+ "optimizer_weight_decay": 1e-05,
169
+ "warmup_ratio": 0.05,
170
+ "use_bf16": true,
171
+ "model_params_fp32": true,
172
+ "image_size": [
173
+ 256,
174
+ 256
175
+ ],
176
+ "tokenizer_assets_repo": null,
177
+ "lora_rank": 0,
178
+ "lora_alpha": 16,
179
+ "lora_dropout": 0.1,
180
+ "lora_full_model": false,
181
+ "video_backend": "decord",
182
+ "balance_dataset_weights": true,
183
+ "balance_trajectory_weights": true,
184
+ "dataset_paths": null,
185
+ "output_dir": "./tmp/gr00t",
186
+ "save_steps": 1000,
187
+ "max_steps": 10000,
188
+ "batch_size": 32,
189
+ "dataloader_num_workers": 8,
190
+ "report_to": "wandb",
191
+ "resume": false
192
+ },
193
+ "reward_model": null,
194
+ "output_dir": "/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline",
195
+ "job_name": "INSIGHT_RD_baseline",
196
+ "resume": false,
197
+ "seed": 42,
198
+ "cudnn_deterministic": false,
199
+ "num_workers": 4,
200
+ "batch_size": 128,
201
+ "prefetch_factor": 4,
202
+ "persistent_workers": true,
203
+ "steps": 60000,
204
+ "env_eval_freq": 20000,
205
+ "log_freq": 200,
206
+ "eval_steps": 0,
207
+ "max_eval_samples": 0,
208
+ "tolerance_s": 0.0001,
209
+ "save_checkpoint": true,
210
+ "save_freq": 20000,
211
+ "use_policy_training_preset": true,
212
+ "optimizer": {
213
+ "type": "adamw",
214
+ "lr": 0.0001,
215
+ "weight_decay": 1e-05,
216
+ "grad_clip_norm": 1.0,
217
+ "betas": [
218
+ 0.9,
219
+ 0.999
220
+ ],
221
+ "eps": 1e-08
222
+ },
223
+ "scheduler": {
224
+ "type": "diffuser",
225
+ "num_warmup_steps": 500,
226
+ "name": "cosine"
227
+ },
228
+ "eval": {
229
+ "n_episodes": 50,
230
+ "batch_size": 44,
231
+ "use_async_envs": true,
232
+ "recording": false,
233
+ "recording_repo_id": null,
234
+ "recording_private": false
235
+ },
236
+ "wandb": {
237
+ "enable": true,
238
+ "disable_artifact": true,
239
+ "project": "lerobot",
240
+ "entity": null,
241
+ "notes": null,
242
+ "run_id": "a3ogbmg4",
243
+ "mode": null,
244
+ "add_tags": true
245
+ },
246
+ "peft": null,
247
+ "job": {
248
+ "target": null,
249
+ "image": "huggingface/lerobot-gpu:latest",
250
+ "timeout": "2d",
251
+ "detach": false,
252
+ "tags": []
253
+ },
254
+ "save_checkpoint_to_hub": false,
255
+ "sample_weighting": null,
256
+ "rename_map": {},
257
+ "checkpoint_path": null
258
+ }
Abs_Joint/Baseline/checkpoints/060000/prompt_manifest.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "prompt_set": "detailed",
4
+ "source_file": "task_prompts.json",
5
+ "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
6
+ "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
7
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
8
+ "dataset_revision": "v3.0",
9
+ "task_count": 13,
10
+ "task_codes": [
11
+ "5f",
12
+ "5g",
13
+ "1ext",
14
+ "3b",
15
+ "5b",
16
+ "3a",
17
+ "3c",
18
+ "5h",
19
+ "5d",
20
+ "3d",
21
+ "5e",
22
+ "5a",
23
+ "5c"
24
+ ],
25
+ "prompts": {
26
+ "5f": "Close the bottle in clockwise direction.",
27
+ "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
28
+ "1ext": "Find the arrow guide and open the indicated drawer.",
29
+ "3b": "Open the door, rotate counter-clockwise and push.",
30
+ "5b": "Open the bottle in counter-clockwise direction.",
31
+ "3a": "Open the door, rotate clockwise and push.",
32
+ "3c": "Open the door, rotate clockwise and pull.",
33
+ "5h": "Close the bottle in counter-clockwise direction.",
34
+ "5d": "Close the bottle in clockwise direction.",
35
+ "3d": "Open the door, rotate counter-clockwise and pull.",
36
+ "5e": "Close the bottle in counter-clockwise direction.",
37
+ "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
38
+ "5c": "Open the bottle in clockwise direction."
39
+ }
40
+ }
Abs_Joint/Baseline/checkpoints/060000/resolved_config.yaml ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
3
+ root: /home/seonho/INSIGHTfixposV4_filtered_multispace_v2
4
+ policy:
5
+ type: groot
6
+ device: cuda
7
+ chunk_size: 16
8
+ n_action_steps: 16
9
+ push_to_hub: false
10
+ tune_llm: false
11
+ tune_visual: false
12
+ tune_top_llm_layers: 0
13
+ tune_projector: true
14
+ tune_diffusion_model: true
15
+ tune_vlln: true
16
+ seed: 42
17
+ batch_size: 128
18
+ steps: 60000
19
+ log_freq: 200
20
+ output_dir: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
21
+ job_name: INSIGHT_RD_baseline
22
+ wandb:
23
+ enable: true
24
+ disable_artifact: true
Abs_Joint/Baseline/checkpoints/060000/source_config.yaml ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # INSIGHT GR00T N1.7 training config
2
+ # INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
3
+ insight:
4
+ # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
5
+ action_space: joint_abs
6
+ # Options: coarse | detailed
7
+ prompt_set: detailed
8
+ save_steps: ["60K"]
9
+
10
+ dataset:
11
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
12
+ root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
13
+
14
+ policy:
15
+ type: groot
16
+ device: cuda
17
+ chunk_size: 16
18
+ n_action_steps: 16
19
+ push_to_hub: false
20
+ # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
21
+ # VLM
22
+ tune_llm: false
23
+ tune_visual: false
24
+ tune_top_llm_layers: 0
25
+ # Action Expert
26
+ tune_projector: true
27
+ tune_diffusion_model: true
28
+ tune_vlln: true
29
+
30
+
31
+ seed: 42
32
+ batch_size: 128
33
+ steps: 60000
34
+ log_freq: 200
35
+ output_dir: "${HOME}/groot_insight/train_ckpt/Abs_Joint/Baseline"
36
+ job_name: INSIGHT_RD_baseline
37
+
38
+ wandb:
39
+ enable: true
40
+ disable_artifact: true
41
+
42
+ # Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
43
+ #
44
+ # dataset:
45
+ # eval_split: 0.0
46
+ # image_transforms:
47
+ # enable: false # default; set true only for train-image augmentation.
48
+ #
49
+ # policy:
50
+ # base_model_path: nvidia/GR00T-N1.7-3B
51
+ # embodiment_tag: new_embodiment
52
+ # use_relative_actions: false
53
+ # use_bf16: true
54
+ # repo_id: Whalswp/INSIGHT # unused while push_to_hub is false
55
+ #
56
+ # save_checkpoint: true
57
+ # use_policy_training_preset: true
58
+ # eval_steps: 0
59
+ # env_eval_freq: 0
60
+ #
61
+ # insight.prompt_set options:
62
+ # coarse: High-level task text that requires following the visual guide.
63
+ # detailed: Direction- and interaction-specific text, including rotation and push/pull details.
Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_param_groups.json ADDED
@@ -0,0 +1,577 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "lr": 0.0,
4
+ "betas": [
5
+ 0.9,
6
+ 0.999
7
+ ],
8
+ "eps": 1e-08,
9
+ "weight_decay": 1e-05,
10
+ "amsgrad": false,
11
+ "maximize": false,
12
+ "foreach": null,
13
+ "capturable": false,
14
+ "differentiable": false,
15
+ "fused": null,
16
+ "decoupled_weight_decay": true,
17
+ "initial_lr": 0.0001,
18
+ "params": [
19
+ 0,
20
+ 1,
21
+ 2,
22
+ 3,
23
+ 4,
24
+ 5,
25
+ 6,
26
+ 7,
27
+ 8,
28
+ 9,
29
+ 10,
30
+ 11,
31
+ 12,
32
+ 13,
33
+ 14,
34
+ 15,
35
+ 16,
36
+ 17,
37
+ 18,
38
+ 19,
39
+ 20,
40
+ 21,
41
+ 22,
42
+ 23,
43
+ 24,
44
+ 25,
45
+ 26,
46
+ 27,
47
+ 28,
48
+ 29,
49
+ 30,
50
+ 31,
51
+ 32,
52
+ 33,
53
+ 34,
54
+ 35,
55
+ 36,
56
+ 37,
57
+ 38,
58
+ 39,
59
+ 40,
60
+ 41,
61
+ 42,
62
+ 43,
63
+ 44,
64
+ 45,
65
+ 46,
66
+ 47,
67
+ 48,
68
+ 49,
69
+ 50,
70
+ 51,
71
+ 52,
72
+ 53,
73
+ 54,
74
+ 55,
75
+ 56,
76
+ 57,
77
+ 58,
78
+ 59,
79
+ 60,
80
+ 61,
81
+ 62,
82
+ 63,
83
+ 64,
84
+ 65,
85
+ 66,
86
+ 67,
87
+ 68,
88
+ 69,
89
+ 70,
90
+ 71,
91
+ 72,
92
+ 73,
93
+ 74,
94
+ 75,
95
+ 76,
96
+ 77,
97
+ 78,
98
+ 79,
99
+ 80,
100
+ 81,
101
+ 82,
102
+ 83,
103
+ 84,
104
+ 85,
105
+ 86,
106
+ 87,
107
+ 88,
108
+ 89,
109
+ 90,
110
+ 91,
111
+ 92,
112
+ 93,
113
+ 94,
114
+ 95,
115
+ 96,
116
+ 97,
117
+ 98,
118
+ 99,
119
+ 100,
120
+ 101,
121
+ 102,
122
+ 103,
123
+ 104,
124
+ 105,
125
+ 106,
126
+ 107,
127
+ 108,
128
+ 109,
129
+ 110,
130
+ 111,
131
+ 112,
132
+ 113,
133
+ 114,
134
+ 115,
135
+ 116,
136
+ 117,
137
+ 118,
138
+ 119,
139
+ 120,
140
+ 121,
141
+ 122,
142
+ 123,
143
+ 124,
144
+ 125,
145
+ 126,
146
+ 127,
147
+ 128,
148
+ 129,
149
+ 130,
150
+ 131,
151
+ 132,
152
+ 133,
153
+ 134,
154
+ 135,
155
+ 136,
156
+ 137,
157
+ 138,
158
+ 139,
159
+ 140,
160
+ 141,
161
+ 142,
162
+ 143,
163
+ 144,
164
+ 145,
165
+ 146,
166
+ 147,
167
+ 148,
168
+ 149,
169
+ 150,
170
+ 151,
171
+ 152,
172
+ 153,
173
+ 154,
174
+ 155,
175
+ 156,
176
+ 157,
177
+ 158,
178
+ 159,
179
+ 160,
180
+ 161,
181
+ 162,
182
+ 163,
183
+ 164,
184
+ 165,
185
+ 166,
186
+ 167,
187
+ 168,
188
+ 169,
189
+ 170,
190
+ 171,
191
+ 172,
192
+ 173,
193
+ 174,
194
+ 175,
195
+ 176,
196
+ 177,
197
+ 178,
198
+ 179,
199
+ 180,
200
+ 181,
201
+ 182,
202
+ 183,
203
+ 184,
204
+ 185,
205
+ 186,
206
+ 187,
207
+ 188,
208
+ 189,
209
+ 190,
210
+ 191,
211
+ 192,
212
+ 193,
213
+ 194,
214
+ 195,
215
+ 196,
216
+ 197,
217
+ 198,
218
+ 199,
219
+ 200,
220
+ 201,
221
+ 202,
222
+ 203,
223
+ 204,
224
+ 205,
225
+ 206,
226
+ 207,
227
+ 208,
228
+ 209,
229
+ 210,
230
+ 211,
231
+ 212,
232
+ 213,
233
+ 214,
234
+ 215,
235
+ 216,
236
+ 217,
237
+ 218,
238
+ 219,
239
+ 220,
240
+ 221,
241
+ 222,
242
+ 223,
243
+ 224,
244
+ 225,
245
+ 226,
246
+ 227,
247
+ 228,
248
+ 229,
249
+ 230,
250
+ 231,
251
+ 232,
252
+ 233,
253
+ 234,
254
+ 235,
255
+ 236,
256
+ 237,
257
+ 238,
258
+ 239,
259
+ 240,
260
+ 241,
261
+ 242,
262
+ 243,
263
+ 244,
264
+ 245,
265
+ 246,
266
+ 247,
267
+ 248,
268
+ 249,
269
+ 250,
270
+ 251,
271
+ 252,
272
+ 253,
273
+ 254,
274
+ 255,
275
+ 256,
276
+ 257,
277
+ 258,
278
+ 259,
279
+ 260,
280
+ 261,
281
+ 262,
282
+ 263,
283
+ 264,
284
+ 265,
285
+ 266
286
+ ]
287
+ },
288
+ {
289
+ "weight_decay": 0.0,
290
+ "lr": 0.0,
291
+ "betas": [
292
+ 0.9,
293
+ 0.999
294
+ ],
295
+ "eps": 1e-08,
296
+ "amsgrad": false,
297
+ "maximize": false,
298
+ "foreach": null,
299
+ "capturable": false,
300
+ "differentiable": false,
301
+ "fused": null,
302
+ "decoupled_weight_decay": true,
303
+ "initial_lr": 0.0001,
304
+ "params": [
305
+ 267,
306
+ 268,
307
+ 269,
308
+ 270,
309
+ 271,
310
+ 272,
311
+ 273,
312
+ 274,
313
+ 275,
314
+ 276,
315
+ 277,
316
+ 278,
317
+ 279,
318
+ 280,
319
+ 281,
320
+ 282,
321
+ 283,
322
+ 284,
323
+ 285,
324
+ 286,
325
+ 287,
326
+ 288,
327
+ 289,
328
+ 290,
329
+ 291,
330
+ 292,
331
+ 293,
332
+ 294,
333
+ 295,
334
+ 296,
335
+ 297,
336
+ 298,
337
+ 299,
338
+ 300,
339
+ 301,
340
+ 302,
341
+ 303,
342
+ 304,
343
+ 305,
344
+ 306,
345
+ 307,
346
+ 308,
347
+ 309,
348
+ 310,
349
+ 311,
350
+ 312,
351
+ 313,
352
+ 314,
353
+ 315,
354
+ 316,
355
+ 317,
356
+ 318,
357
+ 319,
358
+ 320,
359
+ 321,
360
+ 322,
361
+ 323,
362
+ 324,
363
+ 325,
364
+ 326,
365
+ 327,
366
+ 328,
367
+ 329,
368
+ 330,
369
+ 331,
370
+ 332,
371
+ 333,
372
+ 334,
373
+ 335,
374
+ 336,
375
+ 337,
376
+ 338,
377
+ 339,
378
+ 340,
379
+ 341,
380
+ 342,
381
+ 343,
382
+ 344,
383
+ 345,
384
+ 346,
385
+ 347,
386
+ 348,
387
+ 349,
388
+ 350,
389
+ 351,
390
+ 352,
391
+ 353,
392
+ 354,
393
+ 355,
394
+ 356,
395
+ 357,
396
+ 358,
397
+ 359,
398
+ 360,
399
+ 361,
400
+ 362,
401
+ 363,
402
+ 364,
403
+ 365,
404
+ 366,
405
+ 367,
406
+ 368,
407
+ 369,
408
+ 370,
409
+ 371,
410
+ 372,
411
+ 373,
412
+ 374,
413
+ 375,
414
+ 376,
415
+ 377,
416
+ 378,
417
+ 379,
418
+ 380,
419
+ 381,
420
+ 382,
421
+ 383,
422
+ 384,
423
+ 385,
424
+ 386,
425
+ 387,
426
+ 388,
427
+ 389,
428
+ 390,
429
+ 391,
430
+ 392,
431
+ 393,
432
+ 394,
433
+ 395,
434
+ 396,
435
+ 397,
436
+ 398,
437
+ 399,
438
+ 400,
439
+ 401,
440
+ 402,
441
+ 403,
442
+ 404,
443
+ 405,
444
+ 406,
445
+ 407,
446
+ 408,
447
+ 409,
448
+ 410,
449
+ 411,
450
+ 412,
451
+ 413,
452
+ 414,
453
+ 415,
454
+ 416,
455
+ 417,
456
+ 418,
457
+ 419,
458
+ 420,
459
+ 421,
460
+ 422,
461
+ 423,
462
+ 424,
463
+ 425,
464
+ 426,
465
+ 427,
466
+ 428,
467
+ 429,
468
+ 430,
469
+ 431,
470
+ 432,
471
+ 433,
472
+ 434,
473
+ 435,
474
+ 436,
475
+ 437,
476
+ 438,
477
+ 439,
478
+ 440,
479
+ 441,
480
+ 442,
481
+ 443,
482
+ 444,
483
+ 445,
484
+ 446,
485
+ 447,
486
+ 448,
487
+ 449,
488
+ 450,
489
+ 451,
490
+ 452,
491
+ 453,
492
+ 454,
493
+ 455,
494
+ 456,
495
+ 457,
496
+ 458,
497
+ 459,
498
+ 460,
499
+ 461,
500
+ 462,
501
+ 463,
502
+ 464,
503
+ 465,
504
+ 466,
505
+ 467,
506
+ 468,
507
+ 469,
508
+ 470,
509
+ 471,
510
+ 472,
511
+ 473,
512
+ 474,
513
+ 475,
514
+ 476,
515
+ 477,
516
+ 478,
517
+ 479,
518
+ 480,
519
+ 481,
520
+ 482,
521
+ 483,
522
+ 484,
523
+ 485,
524
+ 486,
525
+ 487,
526
+ 488,
527
+ 489,
528
+ 490,
529
+ 491,
530
+ 492,
531
+ 493,
532
+ 494,
533
+ 495,
534
+ 496,
535
+ 497,
536
+ 498,
537
+ 499,
538
+ 500,
539
+ 501,
540
+ 502,
541
+ 503,
542
+ 504,
543
+ 505,
544
+ 506,
545
+ 507,
546
+ 508,
547
+ 509,
548
+ 510,
549
+ 511,
550
+ 512,
551
+ 513,
552
+ 514,
553
+ 515,
554
+ 516,
555
+ 517,
556
+ 518,
557
+ 519,
558
+ 520,
559
+ 521,
560
+ 522,
561
+ 523,
562
+ 524,
563
+ 525,
564
+ 526,
565
+ 527,
566
+ 528,
567
+ 529,
568
+ 530,
569
+ 531,
570
+ 532,
571
+ 533,
572
+ 534,
573
+ 535,
574
+ 536
575
+ ]
576
+ }
577
+ ]
Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8a55b000921729c6ed71b96eead039d9c6d38263bebc1f68a0bf6af8a60609b1
3
+ size 12964276740
Abs_Joint/Baseline/checkpoints/060000/training_state/rng_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4fa151c4df5f27995ad9e7874ec1263c3fe39807321efa6361dad12633080a0
3
+ size 15708
Abs_Joint/Baseline/checkpoints/060000/training_state/scheduler_state.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_lrs": [
3
+ 0.0001,
4
+ 0.0001
5
+ ],
6
+ "last_epoch": 60000,
7
+ "_step_count": 60001,
8
+ "_is_initial": false,
9
+ "_get_lr_called_within_step": false,
10
+ "_last_lr": [
11
+ 0.0,
12
+ 0.0
13
+ ],
14
+ "lr_lambdas": [
15
+ null,
16
+ null
17
+ ]
18
+ }
Abs_Joint/Baseline/checkpoints/060000/training_state/training_step.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "step": 60000,
3
+ "num_processes": 1,
4
+ "batch_size": 128
5
+ }
Abs_Joint/Baseline/prompt_manifest.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "prompt_set": "detailed",
4
+ "source_file": "task_prompts.json",
5
+ "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
6
+ "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
7
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
8
+ "dataset_revision": "v3.0",
9
+ "task_count": 13,
10
+ "task_codes": [
11
+ "5f",
12
+ "5g",
13
+ "1ext",
14
+ "3b",
15
+ "5b",
16
+ "3a",
17
+ "3c",
18
+ "5h",
19
+ "5d",
20
+ "3d",
21
+ "5e",
22
+ "5a",
23
+ "5c"
24
+ ],
25
+ "prompts": {
26
+ "5f": "Close the bottle in clockwise direction.",
27
+ "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
28
+ "1ext": "Find the arrow guide and open the indicated drawer.",
29
+ "3b": "Open the door, rotate counter-clockwise and push.",
30
+ "5b": "Open the bottle in counter-clockwise direction.",
31
+ "3a": "Open the door, rotate clockwise and push.",
32
+ "3c": "Open the door, rotate clockwise and pull.",
33
+ "5h": "Close the bottle in counter-clockwise direction.",
34
+ "5d": "Close the bottle in clockwise direction.",
35
+ "3d": "Open the door, rotate counter-clockwise and pull.",
36
+ "5e": "Close the bottle in counter-clockwise direction.",
37
+ "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
38
+ "5c": "Open the bottle in clockwise direction."
39
+ }
40
+ }
Abs_Joint/Baseline/resolved_config.yaml ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
3
+ root: /home/seonho/INSIGHTfixposV4_filtered_multispace_v2
4
+ policy:
5
+ type: groot
6
+ device: cuda
7
+ chunk_size: 16
8
+ n_action_steps: 16
9
+ push_to_hub: false
10
+ tune_llm: false
11
+ tune_visual: false
12
+ tune_top_llm_layers: 0
13
+ tune_projector: true
14
+ tune_diffusion_model: true
15
+ tune_vlln: true
16
+ seed: 42
17
+ batch_size: 128
18
+ steps: 60000
19
+ log_freq: 200
20
+ output_dir: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
21
+ job_name: INSIGHT_RD_baseline
22
+ wandb:
23
+ enable: true
24
+ disable_artifact: true
Abs_Joint/Baseline/source_config.yaml ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # INSIGHT GR00T N1.7 training config
2
+ # INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
3
+ insight:
4
+ # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
5
+ action_space: joint_abs
6
+ # Options: coarse | detailed
7
+ prompt_set: detailed
8
+ save_steps: ["60K"]
9
+
10
+ dataset:
11
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
12
+ root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
13
+
14
+ policy:
15
+ type: groot
16
+ device: cuda
17
+ chunk_size: 16
18
+ n_action_steps: 16
19
+ push_to_hub: false
20
+ # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
21
+ # VLM
22
+ tune_llm: false
23
+ tune_visual: false
24
+ tune_top_llm_layers: 0
25
+ # Action Expert
26
+ tune_projector: true
27
+ tune_diffusion_model: true
28
+ tune_vlln: true
29
+
30
+
31
+ seed: 42
32
+ batch_size: 128
33
+ steps: 60000
34
+ log_freq: 200
35
+ output_dir: "${HOME}/groot_insight/train_ckpt/Abs_Joint/Baseline"
36
+ job_name: INSIGHT_RD_baseline
37
+
38
+ wandb:
39
+ enable: true
40
+ disable_artifact: true
41
+
42
+ # Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
43
+ #
44
+ # dataset:
45
+ # eval_split: 0.0
46
+ # image_transforms:
47
+ # enable: false # default; set true only for train-image augmentation.
48
+ #
49
+ # policy:
50
+ # base_model_path: nvidia/GR00T-N1.7-3B
51
+ # embodiment_tag: new_embodiment
52
+ # use_relative_actions: false
53
+ # use_bf16: true
54
+ # repo_id: Whalswp/INSIGHT # unused while push_to_hub is false
55
+ #
56
+ # save_checkpoint: true
57
+ # use_policy_training_preset: true
58
+ # eval_steps: 0
59
+ # env_eval_freq: 0
60
+ #
61
+ # insight.prompt_set options:
62
+ # coarse: High-level task text that requires following the visual guide.
63
+ # detailed: Direction- and interaction-specific text, including rotation and push/pull details.
Abs_Joint/Baseline/wandb/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
Abs_Joint/Baseline/wandb/debug.log ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
2
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Configure stats pid to 3896698
3
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log
5
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log
6
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():766] calling init triggers
7
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
8
+ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/seonho/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 44, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
9
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():814] starting backend
10
+ 2026-07-17 15:48:31,110 INFO MainThread:3896698 [wandb_init.py:init():829] sending inform_init request
11
+ 2026-07-17 15:48:31,565 INFO MainThread:3896698 [wandb_init.py:init():834] backend started and connected
12
+ 2026-07-17 15:48:31,568 INFO MainThread:3896698 [wandb_init.py:init():904] updated telemetry
13
+ 2026-07-17 15:48:31,579 INFO MainThread:3896698 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
14
+ 2026-07-17 15:48:32,131 INFO MainThread:3896698 [wandb_init.py:init():972] starting run threads in backend
15
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_console_start():2523] atexit reg
16
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2465] Redirects installed.
19
+ 2026-07-17 15:48:32,267 INFO MainThread:3896698 [wandb_init.py:init():1010] run started, returning control to user process
20
+ 2026-07-20 02:13:16,832 INFO wandb-AsyncioManager-main:3896698 [service_client.py:_forward_responses():122] Reached EOF.
21
+ 2026-07-20 02:13:16,833 INFO wandb-AsyncioManager-main:3896698 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
22
+ 2026-07-20 02:13:16,834 ERROR wandb-AsyncioManager-main:3896698 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
23
+ Traceback (most recent call last):
24
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
25
+ await fn()
26
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
27
+ await asyncio_compat.race(
28
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
29
+ async with open_task_group(race=True) as tg:
30
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
31
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
32
+ await anext(self.gen)
33
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
34
+ await task_group._wait_all(race=race, timeout=exit_timeout)
35
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
36
+ raise exc
37
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
38
+ result = await handle.wait_async(timeout=None)
39
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
40
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
41
+ response = await self._handle.wait_async(timeout=timeout)
42
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
43
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
44
+ raise HandleAbandonedError()
45
+ wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/config.yaml ADDED
@@ -0,0 +1,300 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.27.2
4
+ e:
5
+ pbf296fi8oqpaqrnve886ppcvwjoxaoj:
6
+ args:
7
+ - --config_path=/tmp/insight_lerobot_cfg_s1elc9y_/baseline.yaml
8
+ - --batch_size=128
9
+ codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
10
+ codePathLocal: my_scripts/lerobot_train_yaml.py
11
+ cpu_count: 32
12
+ cpu_count_logical: 64
13
+ cudaVersion: "13.2"
14
+ disk:
15
+ /:
16
+ total: "3778130485248"
17
+ used: "3148893519872"
18
+ executable: /home/seonho/miniconda3/envs/lerobot060_groot/bin/python
19
+ git:
20
+ commit: a970d9f98f189faccce13cf2fb61ccf5d55921f3
21
+ remote: git@github.com:paragon7060/clvla.git
22
+ gpu: NVIDIA A100 80GB PCIe
23
+ gpu_count: 4
24
+ gpu_nvidia:
25
+ - architecture: Ampere
26
+ cudaCores: 6912
27
+ memoryTotal: "85899345920"
28
+ name: NVIDIA A100 80GB PCIe
29
+ uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9
30
+ - architecture: Ampere
31
+ cudaCores: 6912
32
+ memoryTotal: "85899345920"
33
+ name: NVIDIA A100 80GB PCIe
34
+ uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa
35
+ - architecture: Ampere
36
+ cudaCores: 6912
37
+ memoryTotal: "85899345920"
38
+ name: NVIDIA A100 80GB PCIe
39
+ uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac
40
+ - architecture: Ampere
41
+ cudaCores: 6912
42
+ memoryTotal: "85899345920"
43
+ name: NVIDIA A100 80GB PCIe
44
+ uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d
45
+ host: worker1
46
+ memory:
47
+ total: "1081620799488"
48
+ os: Linux-6.8.0-110-generic-x86_64-with-glibc2.35
49
+ program: /home/seonho/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
50
+ python: CPython 3.12.13
51
+ root: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
52
+ startedAt: "2026-07-17T06:48:30.892778Z"
53
+ writerId: pbf296fi8oqpaqrnve886ppcvwjoxaoj
54
+ m: []
55
+ python_version: 3.12.13
56
+ t:
57
+ "1":
58
+ - 1
59
+ - 11
60
+ - 41
61
+ - 49
62
+ - 51
63
+ - 71
64
+ - 83
65
+ - 98
66
+ "2":
67
+ - 1
68
+ - 11
69
+ - 41
70
+ - 49
71
+ - 51
72
+ - 71
73
+ - 83
74
+ - 98
75
+ "3":
76
+ - 13
77
+ - 15
78
+ - 16
79
+ - 61
80
+ "4": 3.12.13
81
+ "5": 0.27.2
82
+ "6": 5.5.4
83
+ "10":
84
+ - 21
85
+ "12": 0.27.2
86
+ "13": linux-x86_64
87
+ batch_size:
88
+ value: 128
89
+ checkpoint_path:
90
+ value: null
91
+ cudnn_deterministic:
92
+ value: false
93
+ dataset:
94
+ value:
95
+ depth_output_unit: mm
96
+ episodes: null
97
+ eval_split: 0
98
+ image_transforms:
99
+ enable: false
100
+ max_num_transforms: 3
101
+ random_order: false
102
+ tfs:
103
+ affine:
104
+ kwargs:
105
+ degrees:
106
+ - -5
107
+ - 5
108
+ translate:
109
+ - 0.05
110
+ - 0.05
111
+ type: RandomAffine
112
+ weight: 1
113
+ brightness:
114
+ kwargs:
115
+ brightness:
116
+ - 0.8
117
+ - 1.2
118
+ type: ColorJitter
119
+ weight: 1
120
+ contrast:
121
+ kwargs:
122
+ contrast:
123
+ - 0.8
124
+ - 1.2
125
+ type: ColorJitter
126
+ weight: 1
127
+ hue:
128
+ kwargs:
129
+ hue:
130
+ - -0.05
131
+ - 0.05
132
+ type: ColorJitter
133
+ weight: 1
134
+ saturation:
135
+ kwargs:
136
+ saturation:
137
+ - 0.5
138
+ - 1.5
139
+ type: ColorJitter
140
+ weight: 1
141
+ sharpness:
142
+ kwargs:
143
+ sharpness:
144
+ - 0.5
145
+ - 1.5
146
+ type: SharpnessJitter
147
+ weight: 1
148
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
149
+ return_uint8: false
150
+ revision: null
151
+ root: /home/seonho/INSIGHTfixposV4_filtered_multispace_v2
152
+ streaming: false
153
+ use_imagenet_stats: true
154
+ video_backend: torchcodec
155
+ env:
156
+ value: null
157
+ env_eval_freq:
158
+ value: 20000
159
+ eval:
160
+ value:
161
+ batch_size: 44
162
+ n_episodes: 50
163
+ recording: false
164
+ recording_private: false
165
+ recording_repo_id: null
166
+ use_async_envs: true
167
+ eval_steps:
168
+ value: 0
169
+ job:
170
+ value:
171
+ detach: false
172
+ image: huggingface/lerobot-gpu:latest
173
+ tags: []
174
+ target: null
175
+ timeout: 2d
176
+ job_name:
177
+ value: INSIGHT_RD_baseline
178
+ log_freq:
179
+ value: 200
180
+ max_eval_samples:
181
+ value: 0
182
+ num_workers:
183
+ value: 4
184
+ optimizer:
185
+ value:
186
+ betas:
187
+ - 0.9
188
+ - 0.999
189
+ eps: 1e-08
190
+ grad_clip_norm: 1
191
+ lr: 0.0001
192
+ type: adamw
193
+ weight_decay: 1e-05
194
+ output_dir:
195
+ value: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
196
+ peft:
197
+ value: null
198
+ persistent_workers:
199
+ value: true
200
+ policy:
201
+ value:
202
+ action_decode_transform: null
203
+ balance_dataset_weights: true
204
+ balance_trajectory_weights: true
205
+ base_model_path: nvidia/GR00T-N1.7-3B
206
+ batch_size: 32
207
+ chunk_size: 16
208
+ dataloader_num_workers: 8
209
+ dataset_paths: null
210
+ device: cuda
211
+ embodiment_tag: new_embodiment
212
+ image_size:
213
+ - 256
214
+ - 256
215
+ license: null
216
+ lora_alpha: 16
217
+ lora_dropout: 0.1
218
+ lora_full_model: false
219
+ lora_rank: 0
220
+ max_action_dim: 132
221
+ max_state_dim: 132
222
+ max_steps: 10000
223
+ model_params_fp32: true
224
+ n_action_steps: 16
225
+ n_obs_steps: 1
226
+ normalization_mapping:
227
+ ACTION: IDENTITY
228
+ STATE: IDENTITY
229
+ VISUAL: IDENTITY
230
+ num_inference_timesteps: null
231
+ optimizer_betas:
232
+ - 0.9
233
+ - 0.999
234
+ optimizer_eps: 1e-08
235
+ optimizer_lr: 0.0001
236
+ optimizer_weight_decay: 1e-05
237
+ output_dir: ./tmp/gr00t
238
+ pretrained_path: null
239
+ pretrained_revision: null
240
+ private: null
241
+ push_to_hub: false
242
+ relative_exclude_joints: []
243
+ repo_id: null
244
+ report_to: wandb
245
+ resume: false
246
+ rtc_ramp_rate: null
247
+ save_steps: 1000
248
+ tags: null
249
+ tokenizer_assets_repo: null
250
+ tune_diffusion_model: true
251
+ tune_llm: false
252
+ tune_projector: true
253
+ tune_top_llm_layers: 0
254
+ tune_visual: false
255
+ tune_vlln: true
256
+ type: groot
257
+ use_amp: false
258
+ use_bf16: true
259
+ use_flash_attention: false
260
+ use_peft: false
261
+ use_relative_actions: false
262
+ video_backend: decord
263
+ warmup_ratio: 0.05
264
+ prefetch_factor:
265
+ value: 4
266
+ resume:
267
+ value: false
268
+ reward_model:
269
+ value: null
270
+ sample_weighting:
271
+ value: null
272
+ save_checkpoint:
273
+ value: true
274
+ save_checkpoint_to_hub:
275
+ value: false
276
+ save_freq:
277
+ value: 20000
278
+ scheduler:
279
+ value:
280
+ name: cosine
281
+ num_warmup_steps: 500
282
+ type: diffuser
283
+ seed:
284
+ value: 42
285
+ steps:
286
+ value: 60000
287
+ tolerance_s:
288
+ value: 0.0001
289
+ use_policy_training_preset:
290
+ value: true
291
+ wandb:
292
+ value:
293
+ add_tags: true
294
+ disable_artifact: true
295
+ enable: true
296
+ entity: null
297
+ mode: null
298
+ notes: null
299
+ project: lerobot
300
+ run_id: null
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/output.log ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO 2026-07-17 15:48:32 db_utils.py:121 Logs will be synced with wandb.
2
+ INFO 2026-07-17 15:48:32 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/a3ogbmg4
3
+ INFO 2026-07-17 15:48:32 ot_train.py:298 Creating dataset
4
+ INFO 2026-07-17 15:48:34 ot_train.py:332 Creating policy
5
+ Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 6565.38it/s]
6
+ `torch_dtype` is deprecated! Use `dtype` instead!
7
+ Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 2551.60it/s]
8
+ INFO 2026-07-17 15:48:38 ot_train.py:405 Creating optimizer and scheduler
9
+ INFO 2026-07-17 15:48:38 ot_train.py:437 Output dir: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
10
+ INFO 2026-07-17 15:48:38 ot_train.py:444 cfg.steps=60000 (60K)
11
+ INFO 2026-07-17 15:48:38 ot_train.py:445 dataset.num_frames=218367 (218K)
12
+ INFO 2026-07-17 15:48:38 ot_train.py:446 dataset.num_episodes=4930
13
+ INFO 2026-07-17 15:48:38 ot_train.py:449 Effective batch size: 128 x 1 = 128
14
+ INFO 2026-07-17 15:48:38 ot_train.py:450 num_learnable_params=1620515968 (2B)
15
+ INFO 2026-07-17 15:48:38 ot_train.py:451 num_total_params=3144016000 (3B)
16
+ Training: 0%| | 0/60000 [00:00<?, ?step/s]INFO 2026-07-17 15:48:38 ot_train.py:602 Start offline training on a fixed dataset, with effective batch size: 128
17
+ Training: 0%| | 200/60000 [11:48<58:04:19, 3.50s/step]INFO 2026-07-17 16:00:27 ot_train.py:646 step:200 smpl:26K ep:578 epch:0.12 loss:0.753 grdn:1.151 lr:2.0e-05 updt_s:2.377 data_s:1.161 smp/s:36 mem_gb:52.70
18
+ Training: 1%| | 400/60000 [23:29<58:09:51, 3.51s/step]INFO 2026-07-17 16:12:08 ot_train.py:646 step:400 smpl:51K ep:1K epch:0.23 loss:0.162 grdn:1.529 lr:6.0e-05 updt_s:2.368 data_s:1.136 smp/s:37 mem_gb:52.76
19
+ Training: 1%| | 600/60000 [35:11<57:39:38, 3.49s/step]INFO 2026-07-17 16:23:50 ot_train.py:646 step:600 smpl:77K ep:2K epch:0.35 loss:0.123 grdn:0.938 lr:9.5e-05 updt_s:2.367 data_s:1.143 smp/s:36 mem_gb:52.76
20
+ Training: 1%|▏ | 800/60000 [46:53<59:16:57, 3.61s/step]INFO 2026-07-17 16:35:32 ot_train.py:646 step:800 smpl:102K ep:2K epch:0.47 loss:0.099 grdn:0.682 lr:1.0e-04 updt_s:2.370 data_s:1.137 smp/s:37 mem_gb:52.76
21
+ Training: 2%|▏ | 1000/60000 [58:35<58:13:04, 3.55s/step]INFO 2026-07-17 16:47:14 ot_train.py:646 step:1K smpl:128K ep:3K epch:0.59 loss:0.089 grdn:0.569 lr:1.0e-04 updt_s:2.367 data_s:1.143 smp/s:36 mem_gb:52.76
22
+ Training: 2%|▏ | 1200/60000 [1:10:22<56:50:49, 3.48s/step]INFO 2026-07-17 16:59:01 ot_train.py:646 step:1K smpl:154K ep:3K epch:0.70 loss:0.082 grdn:0.520 lr:1.0e-04 updt_s:2.368 data_s:1.165 smp/s:36 mem_gb:52.69
23
+ Training: 2%|▏ | 1400/60000 [1:22:08<57:10:38, 3.51s/step]INFO 2026-07-17 17:10:47 ot_train.py:646 step:1K smpl:179K ep:4K epch:0.82 loss:0.076 grdn:0.455 lr:1.0e-04 updt_s:2.378 data_s:1.150 smp/s:36 mem_gb:52.79
24
+ Training: 3%|▎ | 1600/60000 [1:33:51<56:45:55, 3.50s/step]INFO 2026-07-17 17:22:30 ot_train.py:646 step:2K smpl:205K ep:5K epch:0.94 loss:0.073 grdn:0.427 lr:1.0e-04 updt_s:2.369 data_s:1.143 smp/s:36 mem_gb:52.79
25
+ Training: 3%|▎ | 1800/60000 [1:45:33<57:30:15, 3.56s/step]INFO 2026-07-17 17:34:12 ot_train.py:646 step:2K smpl:230K ep:5K epch:1.06 loss:0.070 grdn:0.404 lr:1.0e-04 updt_s:2.370 data_s:1.141 smp/s:36 mem_gb:52.79
26
+ Training: 3%|▎ | 2000/60000 [1:57:15<56:07:26, 3.48s/step]INFO 2026-07-17 17:45:54 ot_train.py:646 step:2K smpl:256K ep:6K epch:1.17 loss:0.069 grdn:0.391 lr:1.0e-04 updt_s:2.365 data_s:1.143 smp/s:36 mem_gb:52.79
27
+ Training: 4%|▎ | 2200/60000 [2:08:59<56:18:20, 3.51s/step]INFO 2026-07-17 17:57:37 ot_train.py:646 step:2K smpl:282K ep:6K epch:1.29 loss:0.066 grdn:0.370 lr:1.0e-04 updt_s:2.370 data_s:1.145 smp/s:36 mem_gb:52.79
28
+ Training: 4%|▍ | 2400/60000 [2:20:41<56:13:18, 3.51s/step]INFO 2026-07-17 18:09:20 ot_train.py:646 step:2K smpl:307K ep:7K epch:1.41 loss:0.065 grdn:0.350 lr:1.0e-04 updt_s:2.355 data_s:1.158 smp/s:36 mem_gb:52.71
29
+ Training: 4%|▍ | 2600/60000 [2:32:24<56:17:48, 3.53s/step]INFO 2026-07-17 18:21:03 ot_train.py:646 step:3K smpl:333K ep:8K epch:1.52 loss:0.063 grdn:0.338 lr:1.0e-04 updt_s:2.362 data_s:1.151 smp/s:36 mem_gb:52.79
30
+ Training: 5%|▍ | 2800/60000 [2:44:08<56:07:25, 3.53s/step]INFO 2026-07-17 18:32:47 ot_train.py:646 step:3K smpl:358K ep:8K epch:1.64 loss:0.060 grdn:0.311 lr:1.0e-04 updt_s:2.366 data_s:1.152 smp/s:36 mem_gb:52.79
31
+ Training: 5%|▌ | 3000/60000 [2:55:52<56:27:17, 3.57s/step]INFO 2026-07-17 18:44:30 ot_train.py:646 step:3K smpl:384K ep:9K epch:1.76 loss:0.062 grdn:0.321 lr:1.0e-04 updt_s:2.365 data_s:1.149 smp/s:36 mem_gb:52.79
32
+ Training: 5%|▌ | 3200/60000 [3:07:35<55:43:02, 3.53s/step]INFO 2026-07-17 18:56:14 ot_train.py:646 step:3K smpl:410K ep:9K epch:1.88 loss:0.060 grdn:0.301 lr:1.0e-04 updt_s:2.373 data_s:1.144 smp/s:36 mem_gb:52.79
33
+ Training: 6%|▌ | 3400/60000 [3:19:20<54:56:01, 3.49s/step]INFO 2026-07-17 19:07:59 ot_train.py:646 step:3K smpl:435K ep:10K epch:1.99 loss:0.060 grdn:0.293 lr:9.9e-05 updt_s:2.364 data_s:1.155 smp/s:36 mem_gb:52.71
34
+ Training: 6%|▌ | 3600/60000 [3:31:01<55:23:52, 3.54s/step]INFO 2026-07-17 19:19:40 ot_train.py:646 step:4K smpl:461K ep:10K epch:2.11 loss:0.057 grdn:0.281 lr:9.9e-05 updt_s:2.362 data_s:1.141 smp/s:37 mem_gb:52.79
35
+ Training: 6%|▋ | 3800/60000 [3:42:42<54:26:57, 3.49s/step]INFO 2026-07-17 19:31:21 ot_train.py:646 step:4K smpl:486K ep:11K epch:2.23 loss:0.057 grdn:0.283 lr:9.9e-05 updt_s:2.362 data_s:1.140 smp/s:37 mem_gb:52.79
36
+ Training: 7%|▋ | 4000/60000 [3:54:22<54:28:49, 3.50s/step]INFO 2026-07-17 19:43:01 ot_train.py:646 step:4K smpl:512K ep:12K epch:2.34 loss:0.057 grdn:0.279 lr:9.9e-05 updt_s:2.359 data_s:1.140 smp/s:37 mem_gb:52.79
37
+ Training: 7%|▋ | 4200/60000 [4:06:15<55:51:35, 3.60s/step]INFO 2026-07-17 19:54:54 ot_train.py:646 step:4K smpl:538K ep:12K epch:2.46 loss:0.055 grdn:0.263 lr:9.9e-05 updt_s:2.391 data_s:1.171 smp/s:36 mem_gb:52.79
38
+ Training: 7%|▋ | 4400/60000 [4:18:09<55:37:34, 3.60s/step]INFO 2026-07-17 20:06:48 ot_train.py:646 step:4K smpl:563K ep:13K epch:2.58 loss:0.055 grdn:0.257 lr:9.9e-05 updt_s:2.395 data_s:1.175 smp/s:36 mem_gb:52.79
39
+ Training: 8%|▊ | 4600/60000 [4:30:03<54:45:26, 3.56s/step]INFO 2026-07-17 20:18:42 ot_train.py:646 step:5K smpl:589K ep:13K epch:2.70 loss:0.055 grdn:0.270 lr:9.9e-05 updt_s:2.385 data_s:1.182 smp/s:36 mem_gb:52.71
40
+ Training: 8%|▊ | 4800/60000 [4:41:51<54:21:08, 3.54s/step]INFO 2026-07-17 20:30:30 ot_train.py:646 step:5K smpl:614K ep:14K epch:2.81 loss:0.055 grdn:0.258 lr:9.9e-05 updt_s:2.381 data_s:1.157 smp/s:36 mem_gb:52.79
41
+ Training: 8%|▊ | 5000/60000 [4:53:40<53:22:02, 3.49s/step]INFO 2026-07-17 20:42:19 ot_train.py:646 step:5K smpl:640K ep:14K epch:2.93 loss:0.053 grdn:0.252 lr:9.9e-05 updt_s:2.383 data_s:1.160 smp/s:36 mem_gb:52.79
42
+ Training: 9%|▊ | 5200/60000 [5:05:22<53:53:15, 3.54s/step]INFO 2026-07-17 20:54:01 ot_train.py:646 step:5K smpl:666K ep:15K epch:3.05 loss:0.053 grdn:0.248 lr:9.9e-05 updt_s:2.364 data_s:1.144 smp/s:36 mem_gb:52.79
43
+ Training: 9%|▉ | 5400/60000 [5:17:06<53:23:36, 3.52s/step]INFO 2026-07-17 21:05:45 ot_train.py:646 step:5K smpl:691K ep:16K epch:3.17 loss:0.052 grdn:0.240 lr:9.8e-05 updt_s:2.371 data_s:1.147 smp/s:36 mem_gb:52.79
44
+ Training: 9%|▉ | 5600/60000 [5:28:48<52:51:47, 3.50s/step]INFO 2026-07-17 21:17:27 ot_train.py:646 step:6K smpl:717K ep:16K epch:3.28 loss:0.051 grdn:0.241 lr:9.8e-05 updt_s:2.363 data_s:1.145 smp/s:36 mem_gb:52.79
45
+ Training: 10%|▉ | 5800/60000 [5:40:31<52:41:14, 3.50s/step]INFO 2026-07-17 21:29:10 ot_train.py:646 step:6K smpl:742K ep:17K epch:3.40 loss:0.052 grdn:0.251 lr:9.8e-05 updt_s:2.354 data_s:1.157 smp/s:36 mem_gb:52.71
46
+ Training: 10%|█ | 6000/60000 [5:52:12<52:32:43, 3.50s/step]INFO 2026-07-17 21:40:51 ot_train.py:646 step:6K smpl:768K ep:17K epch:3.52 loss:0.053 grdn:0.243 lr:9.8e-05 updt_s:2.357 data_s:1.148 smp/s:37 mem_gb:52.79
47
+ Training: 10%|█ | 6200/60000 [6:03:54<52:55:26, 3.54s/step]INFO 2026-07-17 21:52:33 ot_train.py:646 step:6K smpl:794K ep:18K epch:3.63 loss:0.050 grdn:0.230 lr:9.8e-05 updt_s:2.360 data_s:1.149 smp/s:36 mem_gb:52.79
48
+ Training: 11%|█ | 6400/60000 [6:15:37<52:08:13, 3.50s/step]INFO 2026-07-17 22:04:16 ot_train.py:646 step:6K smpl:819K ep:18K epch:3.75 loss:0.051 grdn:0.240 lr:9.8e-05 updt_s:2.364 data_s:1.151 smp/s:36 mem_gb:52.79
49
+ Training: 11%|█ | 6600/60000 [6:27:19<52:01:01, 3.51s/step]INFO 2026-07-17 22:15:58 ot_train.py:646 step:7K smpl:845K ep:19K epch:3.87 loss:0.052 grdn:0.237 lr:9.8e-05 updt_s:2.359 data_s:1.149 smp/s:36 mem_gb:52.79
50
+ Training: 11%|█▏ | 6800/60000 [6:39:02<53:07:11, 3.59s/step]INFO 2026-07-17 22:27:41 ot_train.py:646 step:7K smpl:870K ep:20K epch:3.99 loss:0.050 grdn:0.230 lr:9.7e-05 updt_s:2.352 data_s:1.157 smp/s:36 mem_gb:52.71
51
+ Training: 12%|█▏ | 7000/60000 [6:50:43<51:53:34, 3.52s/step]INFO 2026-07-17 22:39:22 ot_train.py:646 step:7K smpl:896K ep:20K epch:4.10 loss:0.047 grdn:0.215 lr:9.7e-05 updt_s:2.361 data_s:1.145 smp/s:37 mem_gb:52.79
52
+ Training: 12%|█▏ | 7200/60000 [7:02:26<51:08:34, 3.49s/step]INFO 2026-07-17 22:51:05 ot_train.py:646 step:7K smpl:922K ep:21K epch:4.22 loss:0.048 grdn:0.228 lr:9.7e-05 updt_s:2.363 data_s:1.148 smp/s:36 mem_gb:52.79
53
+ Training: 12%|█▏ | 7400/60000 [7:14:07<51:09:59, 3.50s/step]INFO 2026-07-17 23:02:46 ot_train.py:646 step:7K smpl:947K ep:21K epch:4.34 loss:0.049 grdn:0.226 lr:9.7e-05 updt_s:2.363 data_s:1.144 smp/s:37 mem_gb:52.79
54
+ Training: 13%|█▎ | 7600/60000 [7:25:50<50:49:47, 3.49s/step]INFO 2026-07-17 23:14:29 ot_train.py:646 step:8K smpl:973K ep:22K epch:4.45 loss:0.048 grdn:0.216 lr:9.7e-05 updt_s:2.363 data_s:1.147 smp/s:36 mem_gb:52.79
55
+ Training: 13%|█▎ | 7800/60000 [7:37:32<50:54:18, 3.51s/step]INFO 2026-07-17 23:26:11 ot_train.py:646 step:8K smpl:998K ep:23K epch:4.57 loss:0.048 grdn:0.213 lr:9.6e-05 updt_s:2.367 data_s:1.145 smp/s:36 mem_gb:52.79
56
+ Training: 13%|█▎ | 8000/60000 [7:49:15<50:28:09, 3.49s/step]INFO 2026-07-17 23:37:54 ot_train.py:646 step:8K smpl:1M ep:23K epch:4.69 loss:0.048 grdn:0.217 lr:9.6e-05 updt_s:2.354 data_s:1.157 smp/s:36 mem_gb:52.71
57
+ Training: 14%|█▎ | 8200/60000 [8:01:00<50:11:36, 3.49s/step]INFO 2026-07-17 23:49:39 ot_train.py:646 step:8K smpl:1M ep:24K epch:4.81 loss:0.047 grdn:0.218 lr:9.6e-05 updt_s:2.367 data_s:1.155 smp/s:36 mem_gb:52.79
58
+ Training: 14%|█▍ | 8400/60000 [8:12:41<50:05:38, 3.49s/step]INFO 2026-07-18 00:01:20 ot_train.py:646 step:8K smpl:1M ep:24K epch:4.92 loss:0.046 grdn:0.211 lr:9.6e-05 updt_s:2.355 data_s:1.150 smp/s:37 mem_gb:52.79
59
+ Training: 14%|█▍ | 8600/60000 [8:24:23<49:56:32, 3.50s/step]INFO 2026-07-18 00:13:02 ot_train.py:646 step:9K smpl:1M ep:25K epch:5.04 loss:0.047 grdn:0.226 lr:9.6e-05 updt_s:2.361 data_s:1.148 smp/s:36 mem_gb:52.79
60
+ Training: 15%|█▍ | 8800/60000 [8:36:05<49:49:42, 3.50s/step]INFO 2026-07-18 00:24:44 ot_train.py:646 step:9K smpl:1M ep:25K epch:5.16 loss:0.047 grdn:0.218 lr:9.5e-05 updt_s:2.359 data_s:1.149 smp/s:36 mem_gb:52.79
61
+ Training: 15%|█▌ | 9000/60000 [8:47:48<50:33:30, 3.57s/step]INFO 2026-07-18 00:36:27 ot_train.py:646 step:9K smpl:1M ep:26K epch:5.28 loss:0.046 grdn:0.214 lr:9.5e-05 updt_s:2.362 data_s:1.149 smp/s:36 mem_gb:52.79
62
+ Training: 15%|█▌ | 9200/60000 [8:59:30<49:39:40, 3.52s/step]INFO 2026-07-18 00:48:09 ot_train.py:646 step:9K smpl:1M ep:27K epch:5.39 loss:0.045 grdn:0.204 lr:9.5e-05 updt_s:2.356 data_s:1.155 smp/s:36 mem_gb:52.71
63
+ Training: 16%|█▌ | 9400/60000 [9:11:13<48:54:27, 3.48s/step]INFO 2026-07-18 00:59:52 ot_train.py:646 step:9K smpl:1M ep:27K epch:5.51 loss:0.045 grdn:0.208 lr:9.5e-05 updt_s:2.367 data_s:1.146 smp/s:36 mem_gb:52.79
64
+ Training: 16%|█▌ | 9600/60000 [9:22:57<49:47:20, 3.56s/step]INFO 2026-07-18 01:11:36 ot_train.py:646 step:10K smpl:1M ep:28K epch:5.63 loss:0.045 grdn:0.202 lr:9.4e-05 updt_s:2.371 data_s:1.146 smp/s:36 mem_gb:52.79
65
+ Training: 16%|█▋ | 9800/60000 [9:34:40<49:08:31, 3.52s/step]INFO 2026-07-18 01:23:19 ot_train.py:646 step:10K smpl:1M ep:28K epch:5.74 loss:0.045 grdn:0.216 lr:9.4e-05 updt_s:2.368 data_s:1.146 smp/s:36 mem_gb:52.79
66
+ Training: 17%|█▋ | 10000/60000 [9:46:23<48:48:18, 3.51s/step]INFO 2026-07-18 01:35:02 ot_train.py:646 step:10K smpl:1M ep:29K epch:5.86 loss:0.044 grdn:0.203 lr:9.4e-05 updt_s:2.369 data_s:1.144 smp/s:36 mem_gb:52.79
67
+ Training: 17%|█▋ | 10200/60000 [9:58:06<48:43:20, 3.52s/step]INFO 2026-07-18 01:46:45 ot_train.py:646 step:10K smpl:1M ep:29K epch:5.98 loss:0.045 grdn:0.216 lr:9.4e-05 updt_s:2.360 data_s:1.153 smp/s:36 mem_gb:52.71
68
+ Training: 17%|█▋ | 10400/60000 [10:09:47<48:17:51, 3.51s/step]INFO 2026-07-18 01:58:26 ot_train.py:646 step:10K smpl:1M ep:30K epch:6.10 loss:0.043 grdn:0.208 lr:9.3e-05 updt_s:2.362 data_s:1.142 smp/s:37 mem_gb:52.79
69
+ Training: 18%|█▊ | 10600/60000 [10:21:27<48:35:25, 3.54s/step]INFO 2026-07-18 02:10:06 ot_train.py:646 step:11K smpl:1M ep:31K epch:6.21 loss:0.043 grdn:0.203 lr:9.3e-05 updt_s:2.357 data_s:1.141 smp/s:37 mem_gb:52.79
70
+ Training: 18%|█▊ | 10800/60000 [10:33:07<47:45:59, 3.50s/step]INFO 2026-07-18 02:21:46 ot_train.py:646 step:11K smpl:1M ep:31K epch:6.33 loss:0.043 grdn:0.207 lr:9.3e-05 updt_s:2.358 data_s:1.141 smp/s:37 mem_gb:52.79
71
+ Training: 18%|█▊ | 11000/60000 [10:44:49<47:25:01, 3.48s/step]INFO 2026-07-18 02:33:28 ot_train.py:646 step:11K smpl:1M ep:32K epch:6.45 loss:0.042 grdn:0.202 lr:9.3e-05 updt_s:2.361 data_s:1.146 smp/s:36 mem_gb:52.79
72
+ Training: 19%|█▊ | 11200/60000 [10:56:29<47:43:53, 3.52s/step]INFO 2026-07-18 02:45:08 ot_train.py:646 step:11K smpl:1M ep:32K epch:6.57 loss:0.042 grdn:0.204 lr:9.2e-05 updt_s:2.356 data_s:1.142 smp/s:37 mem_gb:52.79
73
+ Training: 19%|█▉ | 11400/60000 [11:08:08<47:23:30, 3.51s/step]INFO 2026-07-18 02:56:47 ot_train.py:646 step:11K smpl:1M ep:33K epch:6.68 loss:0.042 grdn:0.209 lr:9.2e-05 updt_s:2.347 data_s:1.146 smp/s:37 mem_gb:52.71
74
+ Training: 19%|█▉ | 11600/60000 [11:19:49<46:49:30, 3.48s/step]INFO 2026-07-18 03:08:28 ot_train.py:646 step:12K smpl:1M ep:34K epch:6.80 loss:0.041 grdn:0.211 lr:9.2e-05 updt_s:2.358 data_s:1.148 smp/s:37 mem_gb:52.79
75
+ Training: 20%|█▉ | 11800/60000 [11:31:30<47:05:20, 3.52s/step]INFO 2026-07-18 03:20:09 ot_train.py:646 step:12K smpl:2M ep:34K epch:6.92 loss:0.041 grdn:0.202 lr:9.2e-05 updt_s:2.358 data_s:1.145 smp/s:37 mem_gb:52.79
76
+ Training: 20%|██ | 12000/60000 [11:43:11<46:22:08, 3.48s/step]INFO 2026-07-18 03:31:50 ot_train.py:646 step:12K smpl:2M ep:35K epch:7.03 loss:0.041 grdn:0.201 lr:9.1e-05 updt_s:2.361 data_s:1.142 smp/s:37 mem_gb:52.79
77
+ Training: 20%|██ | 12200/60000 [11:54:55<46:48:17, 3.53s/step]INFO 2026-07-18 03:43:34 ot_train.py:646 step:12K smpl:2M ep:35K epch:7.15 loss:0.041 grdn:0.209 lr:9.1e-05 updt_s:2.367 data_s:1.148 smp/s:36 mem_gb:52.79
78
+ Training: 21%|██ | 12400/60000 [12:06:35<46:16:20, 3.50s/step]INFO 2026-07-18 03:55:14 ot_train.py:646 step:12K smpl:2M ep:36K epch:7.27 loss:0.042 grdn:0.213 lr:9.1e-05 updt_s:2.355 data_s:1.145 smp/s:37 mem_gb:52.79
79
+ Training: 21%|██ | 12600/60000 [12:18:17<45:44:14, 3.47s/step]INFO 2026-07-18 04:06:56 ot_train.py:646 step:13K smpl:2M ep:36K epch:7.39 loss:0.041 grdn:0.207 lr:9.0e-05 updt_s:2.355 data_s:1.151 smp/s:37 mem_gb:52.71
80
+ Training: 21%|██▏ | 12800/60000 [12:29:59<45:54:30, 3.50s/step]INFO 2026-07-18 04:18:38 ot_train.py:646 step:13K smpl:2M ep:37K epch:7.50 loss:0.040 grdn:0.202 lr:9.0e-05 updt_s:2.366 data_s:1.144 smp/s:36 mem_gb:52.79
81
+ Training: 22%|██▏ | 13000/60000 [12:41:41<45:48:34, 3.51s/step]INFO 2026-07-18 04:30:20 ot_train.py:646 step:13K smpl:2M ep:38K epch:7.62 loss:0.039 grdn:0.203 lr:9.0e-05 updt_s:2.362 data_s:1.144 smp/s:37 mem_gb:52.79
82
+ Training: 22%|██▏ | 13200/60000 [12:53:22<45:25:39, 3.49s/step]INFO 2026-07-18 04:42:01 ot_train.py:646 step:13K smpl:2M ep:38K epch:7.74 loss:0.039 grdn:0.200 lr:8.9e-05 updt_s:2.359 data_s:1.144 smp/s:37 mem_gb:52.79
83
+ Training: 22%|██▏ | 13400/60000 [13:05:04<45:06:39, 3.48s/step]INFO 2026-07-18 04:53:42 ot_train.py:646 step:13K smpl:2M ep:39K epch:7.85 loss:0.040 grdn:0.198 lr:8.9e-05 updt_s:2.361 data_s:1.147 smp/s:36 mem_gb:52.79
84
+ Training: 23%|██▎ | 13600/60000 [13:16:45<45:18:36, 3.52s/step]INFO 2026-07-18 05:05:24 ot_train.py:646 step:14K smpl:2M ep:39K epch:7.97 loss:0.039 grdn:0.208 lr:8.9e-05 updt_s:2.353 data_s:1.153 smp/s:37 mem_gb:52.71
85
+ Training: 23%|██▎ | 13800/60000 [13:28:27<44:39:19, 3.48s/step]INFO 2026-07-18 05:17:06 ot_train.py:646 step:14K smpl:2M ep:40K epch:8.09 loss:0.039 grdn:0.197 lr:8.8e-05 updt_s:2.367 data_s:1.144 smp/s:36 mem_gb:52.79
86
+ Training: 23%|██▎ | 14000/60000 [13:40:09<44:26:54, 3.48s/step]INFO 2026-07-18 05:28:48 ot_train.py:646 step:14K smpl:2M ep:40K epch:8.21 loss:0.038 grdn:0.198 lr:8.8e-05 updt_s:2.365 data_s:1.144 smp/s:36 mem_gb:52.79
87
+ Training: 24%|██▎ | 14200/60000 [13:51:51<44:12:47, 3.48s/step]INFO 2026-07-18 05:40:30 ot_train.py:646 step:14K smpl:2M ep:41K epch:8.32 loss:0.039 grdn:0.198 lr:8.8e-05 updt_s:2.361 data_s:1.147 smp/s:36 mem_gb:52.79
88
+ Training: 24%|██▍ | 14400/60000 [14:03:34<44:22:55, 3.50s/step]INFO 2026-07-18 05:52:13 ot_train.py:646 step:14K smpl:2M ep:42K epch:8.44 loss:0.039 grdn:0.205 lr:8.7e-05 updt_s:2.364 data_s:1.145 smp/s:36 mem_gb:52.79
89
+ Training: 24%|██▍ | 14600/60000 [14:15:15<44:40:07, 3.54s/step]INFO 2026-07-18 06:03:54 ot_train.py:646 step:15K smpl:2M ep:42K epch:8.56 loss:0.040 grdn:0.204 lr:8.7e-05 updt_s:2.358 data_s:1.148 smp/s:37 mem_gb:52.79
90
+ Training: 25%|██▍ | 14800/60000 [14:26:55<43:33:40, 3.47s/step]INFO 2026-07-18 06:15:34 ot_train.py:646 step:15K smpl:2M ep:43K epch:8.68 loss:0.038 grdn:0.200 lr:8.7e-05 updt_s:2.345 data_s:1.150 smp/s:37 mem_gb:52.71
91
+ Training: 25%|██▌ | 15000/60000 [14:38:34<43:28:06, 3.48s/step]INFO 2026-07-18 06:27:13 ot_train.py:646 step:15K smpl:2M ep:43K epch:8.79 loss:0.038 grdn:0.207 lr:8.6e-05 updt_s:2.356 data_s:1.141 smp/s:37 mem_gb:52.79
92
+ Training: 25%|██▌ | 15200/60000 [14:50:14<43:34:33, 3.50s/step]INFO 2026-07-18 06:38:53 ot_train.py:646 step:15K smpl:2M ep:44K epch:8.91 loss:0.036 grdn:0.192 lr:8.6e-05 updt_s:2.357 data_s:1.141 smp/s:37 mem_gb:52.79
93
+ Training: 26%|██▌ | 15400/60000 [15:01:56<43:33:11, 3.52s/step]INFO 2026-07-18 06:50:35 ot_train.py:646 step:15K smpl:2M ep:45K epch:9.03 loss:0.037 grdn:0.195 lr:8.5e-05 updt_s:2.365 data_s:1.142 smp/s:36 mem_gb:52.79
94
+ Training: 26%|██▌ | 15600/60000 [15:13:36<43:15:44, 3.51s/step]INFO 2026-07-18 07:02:15 ot_train.py:646 step:16K smpl:2M ep:45K epch:9.14 loss:0.038 grdn:0.202 lr:8.5e-05 updt_s:2.357 data_s:1.143 smp/s:37 mem_gb:52.79
95
+ Training: 26%|██▋ | 15800/60000 [15:25:16<42:29:26, 3.46s/step]INFO 2026-07-18 07:13:55 ot_train.py:646 step:16K smpl:2M ep:46K epch:9.26 loss:0.037 grdn:0.200 lr:8.5e-05 updt_s:2.354 data_s:1.141 smp/s:37 mem_gb:52.79
96
+ Training: 27%|██▋ | 16000/60000 [15:36:57<42:37:05, 3.49s/step]INFO 2026-07-18 07:25:36 ot_train.py:646 step:16K smpl:2M ep:46K epch:9.38 loss:0.036 grdn:0.190 lr:8.4e-05 updt_s:2.355 data_s:1.151 smp/s:37 mem_gb:52.71
97
+ Training: 27%|██▋ | 16200/60000 [15:48:38<42:46:14, 3.52s/step]INFO 2026-07-18 07:37:16 ot_train.py:646 step:16K smpl:2M ep:47K epch:9.50 loss:0.036 grdn:0.194 lr:8.4e-05 updt_s:2.358 data_s:1.143 smp/s:37 mem_gb:52.79
98
+ Training: 27%|██▋ | 16400/60000 [16:00:19<42:08:11, 3.48s/step]INFO 2026-07-18 07:48:58 ot_train.py:646 step:16K smpl:2M ep:47K epch:9.61 loss:0.037 grdn:0.198 lr:8.4e-05 updt_s:2.360 data_s:1.145 smp/s:37 mem_gb:52.79
99
+ Training: 28%|██▊ | 16600/60000 [16:12:00<42:03:53, 3.49s/step]INFO 2026-07-18 08:00:39 ot_train.py:646 step:17K smpl:2M ep:48K epch:9.73 loss:0.036 grdn:0.193 lr:8.3e-05 updt_s:2.361 data_s:1.144 smp/s:37 mem_gb:52.79
100
+ Training: 28%|██▊ | 16800/60000 [16:23:42<42:03:45, 3.51s/step]INFO 2026-07-18 08:12:21 ot_train.py:646 step:17K smpl:2M ep:49K epch:9.85 loss:0.036 grdn:0.198 lr:8.3e-05 updt_s:2.363 data_s:1.143 smp/s:37 mem_gb:52.79
101
+ Training: 28%|██▊ | 17000/60000 [16:35:23<41:40:02, 3.49s/step]INFO 2026-07-18 08:24:02 ot_train.py:646 step:17K smpl:2M ep:49K epch:9.96 loss:0.036 grdn:0.210 lr:8.2e-05 updt_s:2.351 data_s:1.152 smp/s:37 mem_gb:52.71
102
+ Training: 29%|██▊ | 17200/60000 [16:47:05<41:25:32, 3.48s/step]INFO 2026-07-18 08:35:44 ot_train.py:646 step:17K smpl:2M ep:50K epch:10.08 loss:0.036 grdn:0.200 lr:8.2e-05 updt_s:2.367 data_s:1.142 smp/s:36 mem_gb:52.79
103
+ Training: 29%|██▉ | 17400/60000 [16:58:46<41:17:48, 3.49s/step]INFO 2026-07-18 08:47:25 ot_train.py:646 step:17K smpl:2M ep:50K epch:10.20 loss:0.035 grdn:0.210 lr:8.2e-05 updt_s:2.364 data_s:1.140 smp/s:37 mem_gb:52.79
104
+ Training: 29%|██▉ | 17600/60000 [17:10:28<41:12:14, 3.50s/step]INFO 2026-07-18 08:59:07 ot_train.py:646 step:18K smpl:2M ep:51K epch:10.32 loss:0.034 grdn:0.192 lr:8.1e-05 updt_s:2.368 data_s:1.143 smp/s:36 mem_gb:52.79
105
+ Training: 30%|██▉ | 17800/60000 [17:22:09<40:44:31, 3.48s/step]INFO 2026-07-18 09:10:47 ot_train.py:646 step:18K smpl:2M ep:51K epch:10.43 loss:0.034 grdn:0.189 lr:8.1e-05 updt_s:2.358 data_s:1.141 smp/s:37 mem_gb:52.79
106
+ Training: 30%|███ | 18000/60000 [17:33:49<40:46:18, 3.49s/step]INFO 2026-07-18 09:22:28 ot_train.py:646 step:18K smpl:2M ep:52K epch:10.55 loss:0.034 grdn:0.189 lr:8.0e-05 updt_s:2.361 data_s:1.140 smp/s:37 mem_gb:52.79
107
+ Training: 30%|███ | 18200/60000 [17:45:30<40:42:39, 3.51s/step]INFO 2026-07-18 09:34:09 ot_train.py:646 step:18K smpl:2M ep:53K epch:10.67 loss:0.034 grdn:0.204 lr:8.0e-05 updt_s:2.357 data_s:1.146 smp/s:37 mem_gb:52.71
108
+ Training: 31%|███ | 18400/60000 [17:57:11<40:20:32, 3.49s/step]INFO 2026-07-18 09:45:50 ot_train.py:646 step:18K smpl:2M ep:53K epch:10.79 loss:0.033 grdn:0.197 lr:7.9e-05 updt_s:2.367 data_s:1.136 smp/s:37 mem_gb:52.79
109
+ Training: 31%|███ | 18600/60000 [18:08:52<40:07:55, 3.49s/step]INFO 2026-07-18 09:57:31 ot_train.py:646 step:19K smpl:2M ep:54K epch:10.90 loss:0.033 grdn:0.205 lr:7.9e-05 updt_s:2.365 data_s:1.138 smp/s:37 mem_gb:52.79
110
+ Training: 31%|███▏ | 18800/60000 [18:20:32<40:09:57, 3.51s/step]INFO 2026-07-18 10:09:11 ot_train.py:646 step:19K smpl:2M ep:54K epch:11.02 loss:0.034 grdn:0.194 lr:7.9e-05 updt_s:2.359 data_s:1.142 smp/s:37 mem_gb:52.79
111
+ Training: 32%|███▏ | 19000/60000 [18:32:12<39:46:27, 3.49s/step]INFO 2026-07-18 10:20:51 ot_train.py:646 step:19K smpl:2M ep:55K epch:11.14 loss:0.033 grdn:0.198 lr:7.8e-05 updt_s:2.356 data_s:1.140 smp/s:37 mem_gb:52.79
112
+ Training: 32%|███▏ | 19200/60000 [18:43:53<41:38:00, 3.67s/step]INFO 2026-07-18 10:32:32 ot_train.py:646 step:19K smpl:2M ep:55K epch:11.25 loss:0.033 grdn:0.195 lr:7.8e-05 updt_s:2.353 data_s:1.149 smp/s:37 mem_gb:52.71
113
+ Training: 32%|███▏ | 19400/60000 [18:55:33<39:28:50, 3.50s/step]INFO 2026-07-18 10:44:12 ot_train.py:646 step:19K smpl:2M ep:56K epch:11.37 loss:0.033 grdn:0.204 lr:7.7e-05 updt_s:2.363 data_s:1.137 smp/s:37 mem_gb:52.79
114
+ Training: 33%|███▎ | 19600/60000 [19:07:12<38:58:54, 3.47s/step]INFO 2026-07-18 10:55:51 ot_train.py:646 step:20K smpl:3M ep:57K epch:11.49 loss:0.032 grdn:0.200 lr:7.7e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
115
+ Training: 33%|███▎ | 19800/60000 [19:18:53<39:04:03, 3.50s/step]INFO 2026-07-18 11:07:32 ot_train.py:646 step:20K smpl:3M ep:57K epch:11.61 loss:0.033 grdn:0.203 lr:7.6e-05 updt_s:2.362 data_s:1.138 smp/s:37 mem_gb:52.79
116
+ Training: 33%|███▎ | 20000/60000 [19:30:35<38:37:54, 3.48s/step]INFO 2026-07-18 11:19:14 ot_train.py:646 step:20K smpl:3M ep:58K epch:11.72 loss:0.033 grdn:0.201 lr:7.6e-05 updt_s:2.368 data_s:1.140 smp/s:36 mem_gb:52.79
117
+ Training: 34%|███▎ | 20200/60000 [19:42:18<38:39:11, 3.50s/step]INFO 2026-07-18 11:30:57 ot_train.py:646 step:20K smpl:3M ep:58K epch:11.84 loss:0.033 grdn:0.197 lr:7.6e-05 updt_s:2.365 data_s:1.151 smp/s:36 mem_gb:52.79
118
+ Training: 34%|███▍ | 20400/60000 [19:53:57<38:28:49, 3.50s/step]INFO 2026-07-18 11:42:36 ot_train.py:646 step:20K smpl:3M ep:59K epch:11.96 loss:0.032 grdn:0.198 lr:7.5e-05 updt_s:2.351 data_s:1.141 smp/s:37 mem_gb:52.71
119
+ Training: 34%|███▍ | 20600/60000 [20:05:36<38:15:38, 3.50s/step]INFO 2026-07-18 11:54:14 ot_train.py:646 step:21K smpl:3M ep:60K epch:12.08 loss:0.031 grdn:0.198 lr:7.5e-05 updt_s:2.355 data_s:1.137 smp/s:37 mem_gb:52.79
120
+ Training: 35%|███▍ | 20800/60000 [20:17:16<38:11:32, 3.51s/step]INFO 2026-07-18 12:05:55 ot_train.py:646 step:21K smpl:3M ep:60K epch:12.19 loss:0.031 grdn:0.194 lr:7.4e-05 updt_s:2.361 data_s:1.139 smp/s:37 mem_gb:52.79
121
+ Training: 35%|███▌ | 21000/60000 [20:28:54<37:49:19, 3.49s/step]INFO 2026-07-18 12:17:33 ot_train.py:646 step:21K smpl:3M ep:61K epch:12.31 loss:0.031 grdn:0.195 lr:7.4e-05 updt_s:2.353 data_s:1.138 smp/s:37 mem_gb:52.79
122
+ Training: 35%|███▌ | 21200/60000 [20:40:34<37:42:59, 3.50s/step]INFO 2026-07-18 12:29:13 ot_train.py:646 step:21K smpl:3M ep:61K epch:12.43 loss:0.032 grdn:0.196 lr:7.3e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
123
+ Training: 36%|███▌ | 21400/60000 [20:52:12<37:26:57, 3.49s/step]INFO 2026-07-18 12:40:51 ot_train.py:646 step:21K smpl:3M ep:62K epch:12.54 loss:0.031 grdn:0.195 lr:7.3e-05 updt_s:2.354 data_s:1.138 smp/s:37 mem_gb:52.79
124
+ Training: 36%|███▌ | 21600/60000 [21:03:52<36:56:06, 3.46s/step]INFO 2026-07-18 12:52:31 ot_train.py:646 step:22K smpl:3M ep:62K epch:12.66 loss:0.030 grdn:0.196 lr:7.2e-05 updt_s:2.346 data_s:1.149 smp/s:37 mem_gb:52.71
125
+ Training: 36%|███▋ | 21800/60000 [21:15:30<36:55:06, 3.48s/step]INFO 2026-07-18 13:04:09 ot_train.py:646 step:22K smpl:3M ep:63K epch:12.78 loss:0.030 grdn:0.204 lr:7.2e-05 updt_s:2.355 data_s:1.135 smp/s:37 mem_gb:52.79
126
+ Training: 37%|███▋ | 22000/60000 [21:27:09<36:42:26, 3.48s/step]INFO 2026-07-18 13:15:48 ot_train.py:646 step:22K smpl:3M ep:64K epch:12.90 loss:0.031 grdn:0.205 lr:7.1e-05 updt_s:2.356 data_s:1.136 smp/s:37 mem_gb:52.79
127
+ Training: 37%|███▋ | 22200/60000 [21:38:49<36:44:20, 3.50s/step]INFO 2026-07-18 13:27:28 ot_train.py:646 step:22K smpl:3M ep:64K epch:13.01 loss:0.030 grdn:0.197 lr:7.1e-05 updt_s:2.359 data_s:1.139 smp/s:37 mem_gb:52.79
128
+ Training: 37%|███▋ | 22400/60000 [21:50:29<36:52:05, 3.53s/step]INFO 2026-07-18 13:39:08 ot_train.py:646 step:22K smpl:3M ep:65K epch:13.13 loss:0.030 grdn:0.197 lr:7.0e-05 updt_s:2.362 data_s:1.137 smp/s:37 mem_gb:52.79
129
+ Training: 38%|███▊ | 22600/60000 [22:02:09<36:30:25, 3.51s/step]INFO 2026-07-18 13:50:48 ot_train.py:646 step:23K smpl:3M ep:65K epch:13.25 loss:0.029 grdn:0.196 lr:7.0e-05 updt_s:2.351 data_s:1.148 smp/s:37 mem_gb:52.71
130
+ Training: 38%|███▊ | 22800/60000 [22:13:47<36:10:33, 3.50s/step]INFO 2026-07-18 14:02:26 ot_train.py:646 step:23K smpl:3M ep:66K epch:13.36 loss:0.029 grdn:0.198 lr:6.9e-05 updt_s:2.354 data_s:1.137 smp/s:37 mem_gb:52.79
131
+ Training: 38%|███▊ | 23000/60000 [22:25:27<35:47:07, 3.48s/step]INFO 2026-07-18 14:14:06 ot_train.py:646 step:23K smpl:3M ep:66K epch:13.48 loss:0.028 grdn:0.191 lr:6.9e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
132
+ Training: 39%|███▊ | 23200/60000 [22:37:06<35:41:50, 3.49s/step]INFO 2026-07-18 14:25:45 ot_train.py:646 step:23K smpl:3M ep:67K epch:13.60 loss:0.028 grdn:0.198 lr:6.8e-05 updt_s:2.359 data_s:1.137 smp/s:37 mem_gb:52.79
133
+ Training: 39%|███▉ | 23400/60000 [22:48:45<35:32:33, 3.50s/step]INFO 2026-07-18 14:37:24 ot_train.py:646 step:23K smpl:3M ep:68K epch:13.72 loss:0.028 grdn:0.192 lr:6.8e-05 updt_s:2.357 data_s:1.136 smp/s:37 mem_gb:52.79
134
+ Training: 39%|███▉ | 23600/60000 [23:00:24<35:12:50, 3.48s/step]INFO 2026-07-18 14:49:03 ot_train.py:646 step:24K smpl:3M ep:68K epch:13.83 loss:0.029 grdn:0.192 lr:6.7e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
135
+ Training: 40%|███▉ | 23800/60000 [23:12:04<35:03:19, 3.49s/step]INFO 2026-07-18 15:00:43 ot_train.py:646 step:24K smpl:3M ep:69K epch:13.95 loss:0.029 grdn:0.201 lr:6.7e-05 updt_s:2.349 data_s:1.146 smp/s:37 mem_gb:52.71
136
+ Training: 40%|████ | 24000/60000 [23:23:42<34:55:14, 3.49s/step]INFO 2026-07-18 15:12:21 ot_train.py:646 step:24K smpl:3M ep:69K epch:14.07 loss:0.027 grdn:0.193 lr:6.6e-05 updt_s:2.355 data_s:1.137 smp/s:37 mem_gb:52.79
137
+ Training: 40%|████ | 24200/60000 [23:35:23<34:44:20, 3.49s/step]INFO 2026-07-18 15:24:02 ot_train.py:646 step:24K smpl:3M ep:70K epch:14.19 loss:0.028 grdn:0.200 lr:6.6e-05 updt_s:2.363 data_s:1.137 smp/s:37 mem_gb:52.79
138
+ Training: 41%|████ | 24400/60000 [23:47:03<34:17:35, 3.47s/step]INFO 2026-07-18 15:35:42 ot_train.py:646 step:24K smpl:3M ep:71K epch:14.30 loss:0.028 grdn:0.195 lr:6.5e-05 updt_s:2.364 data_s:1.134 smp/s:37 mem_gb:52.79
139
+ Training: 41%|████ | 24600/60000 [23:58:42<34:18:20, 3.49s/step]INFO 2026-07-18 15:47:21 ot_train.py:646 step:25K smpl:3M ep:71K epch:14.42 loss:0.028 grdn:0.200 lr:6.5e-05 updt_s:2.360 data_s:1.135 smp/s:37 mem_gb:52.79
140
+ Training: 41%|████▏ | 24800/60000 [24:10:22<34:55:21, 3.57s/step]INFO 2026-07-18 15:59:01 ot_train.py:646 step:25K smpl:3M ep:72K epch:14.54 loss:0.028 grdn:0.201 lr:6.4e-05 updt_s:2.361 data_s:1.136 smp/s:37 mem_gb:52.79
141
+ Training: 42%|████▏ | 25000/60000 [24:22:01<33:50:33, 3.48s/step]INFO 2026-07-18 16:10:40 ot_train.py:646 step:25K smpl:3M ep:72K epch:14.65 loss:0.027 grdn:0.192 lr:6.4e-05 updt_s:2.348 data_s:1.148 smp/s:37 mem_gb:52.71
142
+ Training: 42%|████▏ | 25200/60000 [24:33:41<33:36:15, 3.48s/step]INFO 2026-07-18 16:22:20 ot_train.py:646 step:25K smpl:3M ep:73K epch:14.77 loss:0.026 grdn:0.187 lr:6.3e-05 updt_s:2.357 data_s:1.139 smp/s:37 mem_gb:52.79
143
+ Training: 42%|████▏ | 25400/60000 [24:45:21<33:22:31, 3.47s/step]INFO 2026-07-18 16:34:00 ot_train.py:646 step:25K smpl:3M ep:73K epch:14.89 loss:0.027 grdn:0.199 lr:6.3e-05 updt_s:2.359 data_s:1.139 smp/s:37 mem_gb:52.79
144
+ Training: 43%|████▎ | 25600/60000 [24:57:00<33:22:01, 3.49s/step]INFO 2026-07-18 16:45:39 ot_train.py:646 step:26K smpl:3M ep:74K epch:15.01 loss:0.027 grdn:0.193 lr:6.2e-05 updt_s:2.356 data_s:1.138 smp/s:37 mem_gb:52.79
145
+ Training: 43%|████▎ | 25800/60000 [25:08:39<33:14:07, 3.50s/step]INFO 2026-07-18 16:57:18 ot_train.py:646 step:26K smpl:3M ep:75K epch:15.12 loss:0.026 grdn:0.189 lr:6.2e-05 updt_s:2.356 data_s:1.137 smp/s:37 mem_gb:52.79
146
+ Training: 43%|████▎ | 26000/60000 [25:20:20<33:19:34, 3.53s/step]INFO 2026-07-18 17:08:59 ot_train.py:646 step:26K smpl:3M ep:75K epch:15.24 loss:0.026 grdn:0.192 lr:6.1e-05 updt_s:2.358 data_s:1.147 smp/s:37 mem_gb:52.71
147
+ Training: 44%|████▎ | 26200/60000 [25:32:01<33:02:24, 3.52s/step]INFO 2026-07-18 17:20:40 ot_train.py:646 step:26K smpl:3M ep:76K epch:15.36 loss:0.025 grdn:0.193 lr:6.1e-05 updt_s:2.360 data_s:1.144 smp/s:37 mem_gb:52.79
148
+ Training: 44%|████▍ | 26400/60000 [25:43:40<32:34:24, 3.49s/step]INFO 2026-07-18 17:32:19 ot_train.py:646 step:26K smpl:3M ep:76K epch:15.47 loss:0.026 grdn:0.196 lr:6.0e-05 updt_s:2.356 data_s:1.137 smp/s:37 mem_gb:52.79
149
+ Training: 44%|████▍ | 26600/60000 [25:55:19<32:28:31, 3.50s/step]INFO 2026-07-18 17:43:58 ot_train.py:646 step:27K smpl:3M ep:77K epch:15.59 loss:0.025 grdn:0.191 lr:6.0e-05 updt_s:2.355 data_s:1.138 smp/s:37 mem_gb:52.79
150
+ Training: 45%|████▍ | 26800/60000 [26:06:59<32:08:02, 3.48s/step]INFO 2026-07-18 17:55:38 ot_train.py:646 step:27K smpl:3M ep:77K epch:15.71 loss:0.025 grdn:0.184 lr:5.9e-05 updt_s:2.360 data_s:1.140 smp/s:37 mem_gb:52.79
151
+ Training: 45%|████▌ | 27000/60000 [26:18:40<31:57:35, 3.49s/step]INFO 2026-07-18 18:07:19 ot_train.py:646 step:27K smpl:3M ep:78K epch:15.83 loss:0.026 grdn:0.187 lr:5.9e-05 updt_s:2.364 data_s:1.137 smp/s:37 mem_gb:52.79
152
+ Training: 45%|████▌ | 27200/60000 [26:30:20<31:43:29, 3.48s/step]INFO 2026-07-18 18:18:59 ot_train.py:646 step:27K smpl:3M ep:79K epch:15.94 loss:0.025 grdn:0.190 lr:5.8e-05 updt_s:2.356 data_s:1.144 smp/s:37 mem_gb:52.71
153
+ Training: 46%|████▌ | 27400/60000 [26:42:00<31:27:02, 3.47s/step]INFO 2026-07-18 18:30:38 ot_train.py:646 step:27K smpl:4M ep:79K epch:16.06 loss:0.024 grdn:0.189 lr:5.8e-05 updt_s:2.358 data_s:1.136 smp/s:37 mem_gb:52.79
154
+ Training: 46%|████▌ | 27600/60000 [26:53:41<31:22:41, 3.49s/step]INFO 2026-07-18 18:42:19 ot_train.py:646 step:28K smpl:4M ep:80K epch:16.18 loss:0.025 grdn:0.186 lr:5.7e-05 updt_s:2.363 data_s:1.140 smp/s:37 mem_gb:52.79
155
+ Training: 46%|████▋ | 27800/60000 [27:05:20<31:08:37, 3.48s/step]INFO 2026-07-18 18:53:59 ot_train.py:646 step:28K smpl:4M ep:80K epch:16.30 loss:0.024 grdn:0.187 lr:5.7e-05 updt_s:2.356 data_s:1.138 smp/s:37 mem_gb:52.79
156
+ Training: 47%|████▋ | 28000/60000 [27:16:59<31:07:55, 3.50s/step]INFO 2026-07-18 19:05:38 ot_train.py:646 step:28K smpl:4M ep:81K epch:16.41 loss:0.024 grdn:0.193 lr:5.6e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
157
+ Training: 47%|████▋ | 28200/60000 [27:28:41<30:41:38, 3.47s/step]INFO 2026-07-18 19:17:20 ot_train.py:646 step:28K smpl:4M ep:81K epch:16.53 loss:0.024 grdn:0.193 lr:5.6e-05 updt_s:2.366 data_s:1.142 smp/s:36 mem_gb:52.79
158
+ Training: 47%|████▋ | 28400/60000 [27:40:21<30:36:21, 3.49s/step]INFO 2026-07-18 19:29:00 ot_train.py:646 step:28K smpl:4M ep:82K epch:16.65 loss:0.024 grdn:0.191 lr:5.5e-05 updt_s:2.350 data_s:1.149 smp/s:37 mem_gb:52.71
159
+ Training: 48%|████▊ | 28600/60000 [27:52:01<30:19:17, 3.48s/step]INFO 2026-07-18 19:40:40 ot_train.py:646 step:29K smpl:4M ep:83K epch:16.76 loss:0.023 grdn:0.182 lr:5.5e-05 updt_s:2.357 data_s:1.139 smp/s:37 mem_gb:52.79
160
+ Training: 48%|████▊ | 28800/60000 [28:03:41<30:21:29, 3.50s/step]INFO 2026-07-18 19:52:20 ot_train.py:646 step:29K smpl:4M ep:83K epch:16.88 loss:0.023 grdn:0.185 lr:5.4e-05 updt_s:2.354 data_s:1.144 smp/s:37 mem_gb:52.79
161
+ Training: 48%|████▊ | 29000/60000 [28:15:21<29:56:06, 3.48s/step]INFO 2026-07-18 20:04:00 ot_train.py:646 step:29K smpl:4M ep:84K epch:17.00 loss:0.023 grdn:0.178 lr:5.4e-05 updt_s:2.359 data_s:1.140 smp/s:37 mem_gb:52.79
162
+ Training: 49%|████▊ | 29200/60000 [28:27:02<29:51:44, 3.49s/step]INFO 2026-07-18 20:15:41 ot_train.py:646 step:29K smpl:4M ep:84K epch:17.12 loss:0.023 grdn:0.184 lr:5.3e-05 updt_s:2.366 data_s:1.139 smp/s:37 mem_gb:52.79
163
+ Training: 49%|████▉ | 29400/60000 [28:38:41<29:39:14, 3.49s/step]INFO 2026-07-18 20:27:20 ot_train.py:646 step:29K smpl:4M ep:85K epch:17.23 loss:0.023 grdn:0.188 lr:5.3e-05 updt_s:2.347 data_s:1.146 smp/s:37 mem_gb:52.71
164
+ Training: 49%|████▉ | 29600/60000 [28:50:20<29:39:11, 3.51s/step]INFO 2026-07-18 20:38:59 ot_train.py:646 step:30K smpl:4M ep:86K epch:17.35 loss:0.022 grdn:0.187 lr:5.2e-05 updt_s:2.356 data_s:1.136 smp/s:37 mem_gb:52.79
165
+ Training: 50%|████▉ | 29800/60000 [29:01:59<29:22:18, 3.50s/step]INFO 2026-07-18 20:50:38 ot_train.py:646 step:30K smpl:4M ep:86K epch:17.47 loss:0.022 grdn:0.184 lr:5.1e-05 updt_s:2.355 data_s:1.141 smp/s:37 mem_gb:52.79
166
+ Training: 50%|█████ | 30000/60000 [29:13:39<28:57:38, 3.48s/step]INFO 2026-07-18 21:02:18 ot_train.py:646 step:30K smpl:4M ep:87K epch:17.59 loss:0.022 grdn:0.188 lr:5.1e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
167
+ Training: 50%|█████ | 30200/60000 [29:25:18<28:59:41, 3.50s/step]INFO 2026-07-18 21:13:57 ot_train.py:646 step:30K smpl:4M ep:87K epch:17.70 loss:0.022 grdn:0.181 lr:5.0e-05 updt_s:2.356 data_s:1.141 smp/s:37 mem_gb:52.79
168
+ Training: 51%|█████ | 30400/60000 [29:36:58<28:58:43, 3.52s/step]INFO 2026-07-18 21:25:37 ot_train.py:646 step:30K smpl:4M ep:88K epch:17.82 loss:0.022 grdn:0.191 lr:5.0e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
169
+ Training: 51%|█████ | 30600/60000 [29:48:36<28:34:13, 3.50s/step]INFO 2026-07-18 21:37:15 ot_train.py:646 step:31K smpl:4M ep:88K epch:17.94 loss:0.021 grdn:0.179 lr:4.9e-05 updt_s:2.350 data_s:1.142 smp/s:37 mem_gb:52.71
170
+ Training: 51%|█████▏ | 30800/60000 [30:00:15<28:14:43, 3.48s/step]INFO 2026-07-18 21:48:54 ot_train.py:646 step:31K smpl:4M ep:89K epch:18.05 loss:0.021 grdn:0.186 lr:4.9e-05 updt_s:2.355 data_s:1.138 smp/s:37 mem_gb:52.79
171
+ Training: 52%|█████▏ | 31000/60000 [30:11:55<28:07:42, 3.49s/step]INFO 2026-07-18 22:00:34 ot_train.py:646 step:31K smpl:4M ep:90K epch:18.17 loss:0.021 grdn:0.186 lr:4.8e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
172
+ Training: 52%|█████▏ | 31200/60000 [30:23:35<27:45:41, 3.47s/step]INFO 2026-07-18 22:12:14 ot_train.py:646 step:31K smpl:4M ep:90K epch:18.29 loss:0.021 grdn:0.184 lr:4.8e-05 updt_s:2.361 data_s:1.141 smp/s:37 mem_gb:52.79
173
+ Training: 52%|█████▏ | 31400/60000 [30:35:15<27:36:15, 3.47s/step]INFO 2026-07-18 22:23:54 ot_train.py:646 step:31K smpl:4M ep:91K epch:18.41 loss:0.021 grdn:0.171 lr:4.7e-05 updt_s:2.355 data_s:1.141 smp/s:37 mem_gb:52.79
174
+ Training: 53%|█████▎ | 31600/60000 [30:46:54<27:21:09, 3.47s/step]INFO 2026-07-18 22:35:33 ot_train.py:646 step:32K smpl:4M ep:91K epch:18.52 loss:0.021 grdn:0.178 lr:4.7e-05 updt_s:2.357 data_s:1.139 smp/s:37 mem_gb:52.79
175
+ Training: 53%|█████▎ | 31800/60000 [30:58:34<28:04:45, 3.58s/step]INFO 2026-07-18 22:47:13 ot_train.py:646 step:32K smpl:4M ep:92K epch:18.64 loss:0.020 grdn:0.178 lr:4.6e-05 updt_s:2.350 data_s:1.145 smp/s:37 mem_gb:52.71
176
+ Training: 53%|█████▎ | 32000/60000 [31:10:13<26:58:01, 3.47s/step]INFO 2026-07-18 22:58:52 ot_train.py:646 step:32K smpl:4M ep:92K epch:18.76 loss:0.020 grdn:0.179 lr:4.6e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
177
+ Training: 54%|█████▎ | 32200/60000 [31:21:52<26:49:36, 3.47s/step]INFO 2026-07-18 23:10:31 ot_train.py:646 step:32K smpl:4M ep:93K epch:18.87 loss:0.020 grdn:0.174 lr:4.5e-05 updt_s:2.358 data_s:1.134 smp/s:37 mem_gb:52.79
178
+ Training: 54%|█████▍ | 32400/60000 [31:33:30<27:00:58, 3.52s/step]INFO 2026-07-18 23:22:08 ot_train.py:646 step:32K smpl:4M ep:94K epch:18.99 loss:0.020 grdn:0.179 lr:4.5e-05 updt_s:2.352 data_s:1.134 smp/s:37 mem_gb:52.79
179
+ Training: 54%|█████▍ | 32600/60000 [31:45:07<26:24:06, 3.47s/step]INFO 2026-07-18 23:33:46 ot_train.py:646 step:33K smpl:4M ep:94K epch:19.11 loss:0.020 grdn:0.175 lr:4.4e-05 updt_s:2.353 data_s:1.134 smp/s:37 mem_gb:52.79
180
+ Training: 55%|█████▍ | 32800/60000 [31:56:46<26:16:12, 3.48s/step]INFO 2026-07-18 23:45:24 ot_train.py:646 step:33K smpl:4M ep:95K epch:19.23 loss:0.019 grdn:0.165 lr:4.4e-05 updt_s:2.347 data_s:1.142 smp/s:37 mem_gb:52.71
181
+ Training: 55%|█████▌ | 33000/60000 [32:08:25<26:07:07, 3.48s/step]INFO 2026-07-18 23:57:04 ot_train.py:646 step:33K smpl:4M ep:95K epch:19.34 loss:0.019 grdn:0.170 lr:4.3e-05 updt_s:2.361 data_s:1.133 smp/s:37 mem_gb:52.79
182
+ Training: 55%|█████▌ | 33200/60000 [32:20:04<25:50:57, 3.47s/step]INFO 2026-07-19 00:08:42 ot_train.py:646 step:33K smpl:4M ep:96K epch:19.46 loss:0.019 grdn:0.178 lr:4.3e-05 updt_s:2.359 data_s:1.133 smp/s:37 mem_gb:52.79
183
+ Training: 56%|█████▌ | 33400/60000 [32:31:42<25:46:49, 3.49s/step]INFO 2026-07-19 00:20:21 ot_train.py:646 step:33K smpl:4M ep:97K epch:19.58 loss:0.019 grdn:0.166 lr:4.2e-05 updt_s:2.359 data_s:1.133 smp/s:37 mem_gb:52.79
184
+ Training: 56%|█████▌ | 33600/60000 [32:43:21<25:34:28, 3.49s/step]INFO 2026-07-19 00:32:00 ot_train.py:646 step:34K smpl:4M ep:97K epch:19.70 loss:0.019 grdn:0.171 lr:4.1e-05 updt_s:2.356 data_s:1.134 smp/s:37 mem_gb:52.79
185
+ Training: 56%|█████▋ | 33800/60000 [32:55:01<25:19:38, 3.48s/step]INFO 2026-07-19 00:43:40 ot_train.py:646 step:34K smpl:4M ep:98K epch:19.81 loss:0.019 grdn:0.177 lr:4.1e-05 updt_s:2.362 data_s:1.136 smp/s:37 mem_gb:52.79
186
+ Training: 57%|█████▋ | 34000/60000 [33:06:40<25:51:34, 3.58s/step]INFO 2026-07-19 00:55:19 ot_train.py:646 step:34K smpl:4M ep:98K epch:19.93 loss:0.019 grdn:0.177 lr:4.0e-05 updt_s:2.354 data_s:1.140 smp/s:37 mem_gb:52.71
187
+ Training: 57%|█████▋ | 34200/60000 [33:18:20<25:04:37, 3.50s/step]INFO 2026-07-19 01:06:59 ot_train.py:646 step:34K smpl:4M ep:99K epch:20.05 loss:0.018 grdn:0.162 lr:4.0e-05 updt_s:2.365 data_s:1.135 smp/s:37 mem_gb:52.79
188
+ Training: 57%|█████▋ | 34400/60000 [33:29:59<24:43:44, 3.48s/step]INFO 2026-07-19 01:18:37 ot_train.py:646 step:34K smpl:4M ep:99K epch:20.16 loss:0.018 grdn:0.176 lr:3.9e-05 updt_s:2.358 data_s:1.132 smp/s:37 mem_gb:52.79
189
+ Training: 58%|█████▊ | 34600/60000 [33:41:38<24:38:22, 3.49s/step]INFO 2026-07-19 01:30:17 ot_train.py:646 step:35K smpl:4M ep:100K epch:20.28 loss:0.018 grdn:0.176 lr:3.9e-05 updt_s:2.363 data_s:1.134 smp/s:37 mem_gb:52.79
190
+ Training: 58%|█████▊ | 34800/60000 [33:53:18<24:47:44, 3.54s/step]INFO 2026-07-19 01:41:57 ot_train.py:646 step:35K smpl:4M ep:101K epch:20.40 loss:0.018 grdn:0.175 lr:3.8e-05 updt_s:2.361 data_s:1.135 smp/s:37 mem_gb:52.79
191
+ Training: 58%|█████▊ | 35000/60000 [34:04:54<25:18:25, 3.64s/step]INFO 2026-07-19 01:53:33 ot_train.py:646 step:35K smpl:4M ep:101K epch:20.52 loss:0.018 grdn:0.173 lr:3.8e-05 updt_s:2.339 data_s:1.140 smp/s:37 mem_gb:52.71
192
+ Training: 59%|█████▊ | 35200/60000 [34:16:30<23:58:36, 3.48s/step]INFO 2026-07-19 02:05:09 ot_train.py:646 step:35K smpl:5M ep:102K epch:20.63 loss:0.017 grdn:0.169 lr:3.7e-05 updt_s:2.339 data_s:1.140 smp/s:37 mem_gb:52.79
193
+ Training: 59%|█████▉ | 35400/60000 [34:28:07<23:45:31, 3.48s/step]INFO 2026-07-19 02:16:46 ot_train.py:646 step:35K smpl:5M ep:102K epch:20.75 loss:0.017 grdn:0.181 lr:3.7e-05 updt_s:2.343 data_s:1.138 smp/s:37 mem_gb:52.79
194
+ Training: 59%|█████▉ | 35600/60000 [34:39:43<23:39:48, 3.49s/step]INFO 2026-07-19 02:28:21 ot_train.py:646 step:36K smpl:5M ep:103K epch:20.87 loss:0.017 grdn:0.166 lr:3.6e-05 updt_s:2.340 data_s:1.137 smp/s:37 mem_gb:52.79
195
+ Training: 60%|█████▉ | 35800/60000 [34:51:19<23:17:48, 3.47s/step]INFO 2026-07-19 02:39:58 ot_train.py:646 step:36K smpl:5M ep:103K epch:20.98 loss:0.017 grdn:0.165 lr:3.6e-05 updt_s:2.341 data_s:1.140 smp/s:37 mem_gb:52.79
196
+ Training: 60%|██████ | 36000/60000 [35:02:54<22:58:22, 3.45s/step]INFO 2026-07-19 02:51:33 ot_train.py:646 step:36K smpl:5M ep:104K epch:21.10 loss:0.017 grdn:0.167 lr:3.5e-05 updt_s:2.338 data_s:1.136 smp/s:37 mem_gb:52.79
197
+ Training: 60%|██████ | 36200/60000 [35:14:31<22:57:38, 3.47s/step]INFO 2026-07-19 03:03:10 ot_train.py:646 step:36K smpl:5M ep:105K epch:21.22 loss:0.017 grdn:0.167 lr:3.5e-05 updt_s:2.333 data_s:1.147 smp/s:37 mem_gb:52.71
198
+ Training: 61%|██████ | 36400/60000 [35:26:08<22:43:47, 3.47s/step]INFO 2026-07-19 03:14:47 ot_train.py:646 step:36K smpl:5M ep:105K epch:21.34 loss:0.016 grdn:0.165 lr:3.4e-05 updt_s:2.341 data_s:1.141 smp/s:37 mem_gb:52.79
199
+ Training: 61%|██████ | 36600/60000 [35:37:44<22:40:26, 3.49s/step]INFO 2026-07-19 03:26:23 ot_train.py:646 step:37K smpl:5M ep:106K epch:21.45 loss:0.016 grdn:0.167 lr:3.4e-05 updt_s:2.341 data_s:1.140 smp/s:37 mem_gb:52.79
200
+ Training: 61%|██████▏ | 36800/60000 [35:49:21<22:22:39, 3.47s/step]INFO 2026-07-19 03:38:00 ot_train.py:646 step:37K smpl:5M ep:106K epch:21.57 loss:0.016 grdn:0.160 lr:3.3e-05 updt_s:2.343 data_s:1.140 smp/s:37 mem_gb:52.79
201
+ Training: 62%|██████▏ | 37000/60000 [36:00:57<22:14:00, 3.48s/step]INFO 2026-07-19 03:49:36 ot_train.py:646 step:37K smpl:5M ep:107K epch:21.69 loss:0.016 grdn:0.161 lr:3.3e-05 updt_s:2.340 data_s:1.137 smp/s:37 mem_gb:52.79
202
+ Training: 62%|██████▏ | 37200/60000 [36:12:32<22:03:42, 3.48s/step]INFO 2026-07-19 04:01:11 ot_train.py:646 step:37K smpl:5M ep:108K epch:21.81 loss:0.016 grdn:0.159 lr:3.2e-05 updt_s:2.338 data_s:1.138 smp/s:37 mem_gb:52.79
203
+ Training: 62%|██████▏ | 37400/60000 [36:24:08<21:48:13, 3.47s/step]INFO 2026-07-19 04:12:47 ot_train.py:646 step:37K smpl:5M ep:108K epch:21.92 loss:0.016 grdn:0.170 lr:3.2e-05 updt_s:2.335 data_s:1.143 smp/s:37 mem_gb:52.71
204
+ Training: 63%|██████▎ | 37600/60000 [36:35:45<21:37:15, 3.47s/step]INFO 2026-07-19 04:24:24 ot_train.py:646 step:38K smpl:5M ep:109K epch:22.04 loss:0.015 grdn:0.169 lr:3.1e-05 updt_s:2.344 data_s:1.139 smp/s:37 mem_gb:52.79
205
+ Training: 63%|██████▎ | 37800/60000 [36:47:22<21:18:32, 3.46s/step]INFO 2026-07-19 04:36:01 ot_train.py:646 step:38K smpl:5M ep:109K epch:22.16 loss:0.016 grdn:0.172 lr:3.1e-05 updt_s:2.340 data_s:1.139 smp/s:37 mem_gb:52.79
206
+ Training: 63%|██████▎ | 38000/60000 [36:58:56<21:06:41, 3.45s/step]INFO 2026-07-19 04:47:35 ot_train.py:646 step:38K smpl:5M ep:110K epch:22.27 loss:0.016 grdn:0.160 lr:3.0e-05 updt_s:2.334 data_s:1.135 smp/s:37 mem_gb:52.79
207
+ Training: 64%|██████▎ | 38200/60000 [37:10:32<21:00:52, 3.47s/step]INFO 2026-07-19 04:59:11 ot_train.py:646 step:38K smpl:5M ep:110K epch:22.39 loss:0.016 grdn:0.163 lr:3.0e-05 updt_s:2.339 data_s:1.140 smp/s:37 mem_gb:52.79
208
+ Training: 64%|██████▍ | 38400/60000 [37:22:08<20:48:47, 3.47s/step]INFO 2026-07-19 05:10:47 ot_train.py:646 step:38K smpl:5M ep:111K epch:22.51 loss:0.015 grdn:0.168 lr:2.9e-05 updt_s:2.333 data_s:1.146 smp/s:37 mem_gb:52.71
209
+ Training: 64%|██████▍ | 38600/60000 [37:33:45<21:04:44, 3.55s/step]INFO 2026-07-19 05:22:24 ot_train.py:646 step:39K smpl:5M ep:112K epch:22.63 loss:0.015 grdn:0.162 lr:2.9e-05 updt_s:2.347 data_s:1.136 smp/s:37 mem_gb:52.79
210
+ Training: 65%|██████▍ | 38800/60000 [37:45:24<20:52:09, 3.54s/step]INFO 2026-07-19 05:34:02 ot_train.py:646 step:39K smpl:5M ep:112K epch:22.74 loss:0.015 grdn:0.172 lr:2.8e-05 updt_s:2.350 data_s:1.141 smp/s:37 mem_gb:52.79
211
+ Training: 65%|██████▌ | 39000/60000 [37:57:14<21:16:02, 3.65s/step]INFO 2026-07-19 05:45:53 ot_train.py:646 step:39K smpl:5M ep:113K epch:22.86 loss:0.015 grdn:0.159 lr:2.8e-05 updt_s:2.387 data_s:1.164 smp/s:36 mem_gb:52.79
212
+ Training: 65%|██████▌ | 39200/60000 [38:09:12<21:11:17, 3.67s/step]INFO 2026-07-19 05:57:50 ot_train.py:646 step:39K smpl:5M ep:113K epch:22.98 loss:0.015 grdn:0.165 lr:2.7e-05 updt_s:2.404 data_s:1.179 smp/s:36 mem_gb:52.79
213
+ Training: 66%|██████▌ | 39400/60000 [38:21:07<20:36:15, 3.60s/step]INFO 2026-07-19 06:09:46 ot_train.py:646 step:39K smpl:5M ep:114K epch:23.10 loss:0.014 grdn:0.163 lr:2.7e-05 updt_s:2.396 data_s:1.178 smp/s:36 mem_gb:52.79
214
+ Training: 66%|██████▌ | 39600/60000 [38:33:05<20:00:18, 3.53s/step]INFO 2026-07-19 06:21:44 ot_train.py:646 step:40K smpl:5M ep:114K epch:23.21 loss:0.014 grdn:0.164 lr:2.7e-05 updt_s:2.388 data_s:1.201 smp/s:36 mem_gb:52.71
215
+ Training: 66%|██████▋ | 39800/60000 [38:44:56<20:11:14, 3.60s/step]INFO 2026-07-19 06:33:35 ot_train.py:646 step:40K smpl:5M ep:115K epch:23.33 loss:0.013 grdn:0.156 lr:2.6e-05 updt_s:2.383 data_s:1.169 smp/s:36 mem_gb:52.79
216
+ Training: 67%|██████▋ | 40000/60000 [38:56:44<19:14:46, 3.46s/step]INFO 2026-07-19 06:45:22 ot_train.py:646 step:40K smpl:5M ep:116K epch:23.45 loss:0.014 grdn:0.155 lr:2.6e-05 updt_s:2.373 data_s:1.163 smp/s:36 mem_gb:52.79
217
+ Training: 67%|██████▋ | 40200/60000 [39:08:23<19:07:30, 3.48s/step]INFO 2026-07-19 06:57:02 ot_train.py:646 step:40K smpl:5M ep:116K epch:23.56 loss:0.013 grdn:0.156 lr:2.5e-05 updt_s:2.352 data_s:1.144 smp/s:37 mem_gb:52.79
218
+ Training: 67%|██████▋ | 40400/60000 [39:20:21<19:45:19, 3.63s/step]INFO 2026-07-19 07:09:00 ot_train.py:646 step:40K smpl:5M ep:117K epch:23.68 loss:0.014 grdn:0.170 lr:2.5e-05 updt_s:2.419 data_s:1.167 smp/s:36 mem_gb:52.79
219
+ Training: 68%|██████▊ | 40600/60000 [39:32:22<19:50:01, 3.68s/step]INFO 2026-07-19 07:21:01 ot_train.py:646 step:41K smpl:5M ep:117K epch:23.80 loss:0.014 grdn:0.161 lr:2.4e-05 updt_s:2.431 data_s:1.173 smp/s:36 mem_gb:52.79
220
+ Training: 68%|██████▊ | 40800/60000 [39:44:15<19:03:35, 3.57s/step]INFO 2026-07-19 07:32:54 ot_train.py:646 step:41K smpl:5M ep:118K epch:23.92 loss:0.013 grdn:0.171 lr:2.4e-05 updt_s:2.384 data_s:1.180 smp/s:36 mem_gb:52.71
221
+ Training: 68%|██████▊ | 41000/60000 [39:56:10<18:39:17, 3.53s/step]INFO 2026-07-19 07:44:49 ot_train.py:646 step:41K smpl:5M ep:118K epch:24.03 loss:0.013 grdn:0.159 lr:2.3e-05 updt_s:2.393 data_s:1.178 smp/s:36 mem_gb:52.79
222
+ Training: 69%|██████▊ | 41200/60000 [40:08:03<19:01:12, 3.64s/step]INFO 2026-07-19 07:56:42 ot_train.py:646 step:41K smpl:5M ep:119K epch:24.15 loss:0.013 grdn:0.156 lr:2.3e-05 updt_s:2.392 data_s:1.172 smp/s:36 mem_gb:52.79
223
+ Training: 69%|██████▉ | 41400/60000 [40:19:44<18:01:40, 3.49s/step]INFO 2026-07-19 08:08:23 ot_train.py:646 step:41K smpl:5M ep:120K epch:24.27 loss:0.013 grdn:0.164 lr:2.2e-05 updt_s:2.356 data_s:1.147 smp/s:37 mem_gb:52.79
224
+ Training: 69%|██████▉ | 41600/60000 [40:31:21<17:44:00, 3.47s/step]INFO 2026-07-19 08:20:00 ot_train.py:646 step:42K smpl:5M ep:120K epch:24.38 loss:0.013 grdn:0.159 lr:2.2e-05 updt_s:2.338 data_s:1.144 smp/s:37 mem_gb:52.79
225
+ Training: 70%|██████▉ | 41800/60000 [40:42:59<17:42:50, 3.50s/step]INFO 2026-07-19 08:31:38 ot_train.py:646 step:42K smpl:5M ep:121K epch:24.50 loss:0.013 grdn:0.154 lr:2.2e-05 updt_s:2.338 data_s:1.154 smp/s:37 mem_gb:52.71
226
+ Training: 70%|███████ | 42000/60000 [40:54:38<17:23:59, 3.48s/step]INFO 2026-07-19 08:43:17 ot_train.py:646 step:42K smpl:5M ep:121K epch:24.62 loss:0.012 grdn:0.151 lr:2.1e-05 updt_s:2.345 data_s:1.145 smp/s:37 mem_gb:52.79
227
+ Training: 70%|███████ | 42200/60000 [41:06:15<17:08:28, 3.47s/step]INFO 2026-07-19 08:54:54 ot_train.py:646 step:42K smpl:5M ep:122K epch:24.74 loss:0.012 grdn:0.154 lr:2.1e-05 updt_s:2.342 data_s:1.144 smp/s:37 mem_gb:52.79
228
+ Training: 71%|███████ | 42400/60000 [41:17:51<17:02:29, 3.49s/step]INFO 2026-07-19 09:06:30 ot_train.py:646 step:42K smpl:5M ep:123K epch:24.85 loss:0.012 grdn:0.156 lr:2.0e-05 updt_s:2.335 data_s:1.142 smp/s:37 mem_gb:52.79
229
+ Training: 71%|███████ | 42600/60000 [41:29:29<16:43:06, 3.46s/step]INFO 2026-07-19 09:18:08 ot_train.py:646 step:43K smpl:5M ep:123K epch:24.97 loss:0.012 grdn:0.156 lr:2.0e-05 updt_s:2.346 data_s:1.143 smp/s:37 mem_gb:52.79
230
+ Training: 71%|███████▏ | 42800/60000 [41:41:06<16:37:46, 3.48s/step]INFO 2026-07-19 09:29:45 ot_train.py:646 step:43K smpl:5M ep:124K epch:25.09 loss:0.012 grdn:0.152 lr:1.9e-05 updt_s:2.341 data_s:1.143 smp/s:37 mem_gb:52.79
231
+ Training: 72%|███████▏ | 43000/60000 [41:52:44<16:22:38, 3.47s/step]INFO 2026-07-19 09:41:23 ot_train.py:646 step:43K smpl:6M ep:124K epch:25.21 loss:0.012 grdn:0.158 lr:1.9e-05 updt_s:2.335 data_s:1.150 smp/s:37 mem_gb:52.71
232
+ Training: 72%|███████▏ | 43200/60000 [42:04:21<16:12:44, 3.47s/step]INFO 2026-07-19 09:53:00 ot_train.py:646 step:43K smpl:6M ep:125K epch:25.32 loss:0.012 grdn:0.163 lr:1.9e-05 updt_s:2.346 data_s:1.141 smp/s:37 mem_gb:52.79
233
+ Training: 72%|███████▏ | 43400/60000 [42:15:59<16:03:56, 3.48s/step]INFO 2026-07-19 10:04:38 ot_train.py:646 step:43K smpl:6M ep:125K epch:25.44 loss:0.012 grdn:0.152 lr:1.8e-05 updt_s:2.347 data_s:1.139 smp/s:37 mem_gb:52.79
234
+ Training: 73%|███████▎ | 43600/60000 [42:27:38<16:21:51, 3.59s/step]INFO 2026-07-19 10:16:17 ot_train.py:646 step:44K smpl:6M ep:126K epch:25.56 loss:0.011 grdn:0.153 lr:1.8e-05 updt_s:2.351 data_s:1.142 smp/s:37 mem_gb:52.79
235
+ Training: 73%|███████▎ | 43800/60000 [42:39:38<16:17:38, 3.62s/step]INFO 2026-07-19 10:28:17 ot_train.py:646 step:44K smpl:6M ep:127K epch:25.67 loss:0.012 grdn:0.157 lr:1.7e-05 updt_s:2.425 data_s:1.171 smp/s:36 mem_gb:52.79
236
+ Training: 73%|███████▎ | 44000/60000 [42:51:37<15:49:05, 3.56s/step]INFO 2026-07-19 10:40:16 ot_train.py:646 step:44K smpl:6M ep:127K epch:25.79 loss:0.011 grdn:0.153 lr:1.7e-05 updt_s:2.422 data_s:1.171 smp/s:36 mem_gb:52.79
237
+ Training: 74%|███████▎ | 44200/60000 [43:03:28<15:37:42, 3.56s/step]INFO 2026-07-19 10:52:07 ot_train.py:646 step:44K smpl:6M ep:128K epch:25.91 loss:0.011 grdn:0.155 lr:1.7e-05 updt_s:2.376 data_s:1.175 smp/s:36 mem_gb:52.71
238
+ Training: 74%|███████▍ | 44400/60000 [43:15:16<15:19:18, 3.54s/step]INFO 2026-07-19 11:03:55 ot_train.py:646 step:44K smpl:6M ep:128K epch:26.03 loss:0.011 grdn:0.158 lr:1.6e-05 updt_s:2.378 data_s:1.163 smp/s:36 mem_gb:52.79
239
+ Training: 74%|███████▍ | 44600/60000 [43:27:08<15:19:23, 3.58s/step]INFO 2026-07-19 11:15:47 ot_train.py:646 step:45K smpl:6M ep:129K epch:26.14 loss:0.011 grdn:0.156 lr:1.6e-05 updt_s:2.393 data_s:1.165 smp/s:36 mem_gb:52.79
240
+ Training: 75%|███████▍ | 44800/60000 [43:38:51<14:50:54, 3.52s/step]INFO 2026-07-19 11:27:30 ot_train.py:646 step:45K smpl:6M ep:129K epch:26.26 loss:0.011 grdn:0.157 lr:1.5e-05 updt_s:2.368 data_s:1.143 smp/s:36 mem_gb:52.79
241
+ Training: 75%|███████▌ | 45000/60000 [43:50:34<14:40:28, 3.52s/step]INFO 2026-07-19 11:39:13 ot_train.py:646 step:45K smpl:6M ep:130K epch:26.38 loss:0.011 grdn:0.152 lr:1.5e-05 updt_s:2.370 data_s:1.141 smp/s:36 mem_gb:52.79
242
+ Training: 75%|███████▌ | 45200/60000 [44:02:16<14:31:37, 3.53s/step]INFO 2026-07-19 11:50:55 ot_train.py:646 step:45K smpl:6M ep:131K epch:26.49 loss:0.011 grdn:0.154 lr:1.5e-05 updt_s:2.359 data_s:1.152 smp/s:36 mem_gb:52.71
243
+ Training: 76%|███████▌ | 45400/60000 [44:13:57<14:20:39, 3.54s/step]INFO 2026-07-19 12:02:36 ot_train.py:646 step:45K smpl:6M ep:131K epch:26.61 loss:0.011 grdn:0.149 lr:1.4e-05 updt_s:2.361 data_s:1.138 smp/s:37 mem_gb:52.79
244
+ Training: 76%|███████▌ | 45600/60000 [44:25:36<14:01:37, 3.51s/step]INFO 2026-07-19 12:14:15 ot_train.py:646 step:46K smpl:6M ep:132K epch:26.73 loss:0.010 grdn:0.148 lr:1.4e-05 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
245
+ Training: 76%|███████▋ | 45800/60000 [44:37:16<13:53:10, 3.52s/step]INFO 2026-07-19 12:25:55 ot_train.py:646 step:46K smpl:6M ep:132K epch:26.85 loss:0.011 grdn:0.156 lr:1.4e-05 updt_s:2.359 data_s:1.141 smp/s:37 mem_gb:52.79
246
+ Training: 77%|███████▋ | 46000/60000 [44:48:55<13:37:09, 3.50s/step]INFO 2026-07-19 12:37:34 ot_train.py:646 step:46K smpl:6M ep:133K epch:26.96 loss:0.011 grdn:0.153 lr:1.3e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
247
+ Training: 77%|███████▋ | 46200/60000 [45:00:35<13:21:22, 3.48s/step]INFO 2026-07-19 12:49:14 ot_train.py:646 step:46K smpl:6M ep:134K epch:27.08 loss:0.011 grdn:0.163 lr:1.3e-05 updt_s:2.355 data_s:1.142 smp/s:37 mem_gb:52.79
248
+ Training: 77%|███████▋ | 46400/60000 [45:12:15<13:07:43, 3.48s/step]INFO 2026-07-19 13:00:53 ot_train.py:646 step:46K smpl:6M ep:134K epch:27.20 loss:0.010 grdn:0.149 lr:1.3e-05 updt_s:2.345 data_s:1.151 smp/s:37 mem_gb:52.71
249
+ Training: 78%|███████▊ | 46600/60000 [45:23:54<13:01:30, 3.50s/step]INFO 2026-07-19 13:12:33 ot_train.py:646 step:47K smpl:6M ep:135K epch:27.32 loss:0.010 grdn:0.153 lr:1.2e-05 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
250
+ Training: 78%|███████▊ | 46800/60000 [45:35:33<12:45:56, 3.48s/step]INFO 2026-07-19 13:24:12 ot_train.py:646 step:47K smpl:6M ep:135K epch:27.43 loss:0.010 grdn:0.146 lr:1.2e-05 updt_s:2.358 data_s:1.138 smp/s:37 mem_gb:52.79
251
+ Training: 78%|███████▊ | 47000/60000 [45:47:14<12:36:07, 3.49s/step]INFO 2026-07-19 13:35:53 ot_train.py:646 step:47K smpl:6M ep:136K epch:27.55 loss:0.010 grdn:0.154 lr:1.1e-05 updt_s:2.364 data_s:1.137 smp/s:37 mem_gb:52.79
252
+ Training: 79%|███████▊ | 47200/60000 [45:58:52<12:20:29, 3.47s/step]INFO 2026-07-19 13:47:31 ot_train.py:646 step:47K smpl:6M ep:136K epch:27.67 loss:0.010 grdn:0.146 lr:1.1e-05 updt_s:2.357 data_s:1.135 smp/s:37 mem_gb:52.79
253
+ Training: 79%|███████▉ | 47400/60000 [46:10:31<12:10:46, 3.48s/step]INFO 2026-07-19 13:59:10 ot_train.py:646 step:47K smpl:6M ep:137K epch:27.78 loss:0.010 grdn:0.145 lr:1.1e-05 updt_s:2.355 data_s:1.136 smp/s:37 mem_gb:52.79
254
+ Training: 79%|███████▉ | 47600/60000 [46:22:09<11:59:36, 3.48s/step]INFO 2026-07-19 14:10:48 ot_train.py:646 step:48K smpl:6M ep:138K epch:27.90 loss:0.010 grdn:0.149 lr:1.0e-05 updt_s:2.345 data_s:1.144 smp/s:37 mem_gb:52.71
255
+ Training: 80%|███████▉ | 47800/60000 [46:33:48<11:44:59, 3.47s/step]INFO 2026-07-19 14:22:27 ot_train.py:646 step:48K smpl:6M ep:138K epch:28.02 loss:0.009 grdn:0.150 lr:1.0e-05 updt_s:2.358 data_s:1.135 smp/s:37 mem_gb:52.79
256
+ Training: 80%|████████ | 48000/60000 [46:45:27<11:40:44, 3.50s/step]INFO 2026-07-19 14:34:06 ot_train.py:646 step:48K smpl:6M ep:139K epch:28.14 loss:0.009 grdn:0.153 lr:9.9e-06 updt_s:2.359 data_s:1.134 smp/s:37 mem_gb:52.79
257
+ Training: 80%|████████ | 48200/60000 [46:57:06<11:25:32, 3.49s/step]INFO 2026-07-19 14:45:45 ot_train.py:646 step:48K smpl:6M ep:139K epch:28.25 loss:0.009 grdn:0.153 lr:9.5e-06 updt_s:2.359 data_s:1.137 smp/s:37 mem_gb:52.79
258
+ Training: 81%|████████ | 48400/60000 [47:08:46<11:17:40, 3.51s/step]INFO 2026-07-19 14:57:25 ot_train.py:646 step:48K smpl:6M ep:140K epch:28.37 loss:0.009 grdn:0.152 lr:9.2e-06 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
259
+ Training: 81%|████████ | 48600/60000 [47:20:27<11:04:46, 3.50s/step]INFO 2026-07-19 15:09:06 ot_train.py:646 step:49K smpl:6M ep:140K epch:28.49 loss:0.009 grdn:0.152 lr:8.9e-06 updt_s:2.361 data_s:1.145 smp/s:37 mem_gb:52.71
260
+ Training: 81%|████████▏ | 48800/60000 [47:32:08<10:59:00, 3.53s/step]INFO 2026-07-19 15:20:46 ot_train.py:646 step:49K smpl:6M ep:141K epch:28.61 loss:0.009 grdn:0.141 lr:8.6e-06 updt_s:2.365 data_s:1.134 smp/s:37 mem_gb:52.79
261
+ Training: 82%|████████▏ | 49000/60000 [47:43:46<10:45:43, 3.52s/step]INFO 2026-07-19 15:32:25 ot_train.py:646 step:49K smpl:6M ep:142K epch:28.72 loss:0.009 grdn:0.149 lr:8.3e-06 updt_s:2.358 data_s:1.133 smp/s:37 mem_gb:52.79
262
+ Training: 82%|████████▏ | 49200/60000 [47:55:25<10:31:36, 3.51s/step]INFO 2026-07-19 15:44:04 ot_train.py:646 step:49K smpl:6M ep:142K epch:28.84 loss:0.009 grdn:0.147 lr:8.1e-06 updt_s:2.360 data_s:1.136 smp/s:37 mem_gb:52.79
263
+ Training: 82%|████████▏ | 49400/60000 [48:07:06<10:20:20, 3.51s/step]INFO 2026-07-19 15:55:44 ot_train.py:646 step:49K smpl:6M ep:143K epch:28.96 loss:0.009 grdn:0.144 lr:7.8e-06 updt_s:2.362 data_s:1.138 smp/s:37 mem_gb:52.79
264
+ Training: 83%|████████▎ | 49600/60000 [48:18:44<10:08:32, 3.51s/step]INFO 2026-07-19 16:07:22 ot_train.py:646 step:50K smpl:6M ep:143K epch:29.07 loss:0.009 grdn:0.144 lr:7.5e-06 updt_s:2.351 data_s:1.137 smp/s:37 mem_gb:52.79
265
+ Training: 83%|████████▎ | 49800/60000 [48:30:22<9:51:39, 3.48s/step]INFO 2026-07-19 16:19:01 ot_train.py:646 step:50K smpl:6M ep:144K epch:29.19 loss:0.009 grdn:0.147 lr:7.2e-06 updt_s:2.346 data_s:1.144 smp/s:37 mem_gb:52.71
266
+ Training: 83%|████████▎ | 50000/60000 [48:42:01<9:41:47, 3.49s/step]INFO 2026-07-19 16:30:40 ot_train.py:646 step:50K smpl:6M ep:144K epch:29.31 loss:0.009 grdn:0.150 lr:6.9e-06 updt_s:2.355 data_s:1.139 smp/s:37 mem_gb:52.79
267
+ Training: 84%|████████▎ | 50200/60000 [48:53:40<9:33:55, 3.51s/step]INFO 2026-07-19 16:42:19 ot_train.py:646 step:50K smpl:6M ep:145K epch:29.43 loss:0.008 grdn:0.142 lr:6.7e-06 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
268
+ Training: 84%|████████▍ | 50400/60000 [49:05:19<9:17:31, 3.48s/step]INFO 2026-07-19 16:53:57 ot_train.py:646 step:50K smpl:6M ep:146K epch:29.54 loss:0.008 grdn:0.147 lr:6.4e-06 updt_s:2.354 data_s:1.137 smp/s:37 mem_gb:52.79
269
+ Training: 84%|████████▍ | 50600/60000 [49:16:58<9:07:37, 3.50s/step]INFO 2026-07-19 17:05:37 ot_train.py:646 step:51K smpl:6M ep:146K epch:29.66 loss:0.009 grdn:0.156 lr:6.2e-06 updt_s:2.356 data_s:1.138 smp/s:37 mem_gb:52.79
270
+ Training: 85%|████████▍ | 50800/60000 [49:28:36<8:52:51, 3.48s/step]INFO 2026-07-19 17:17:15 ot_train.py:646 step:51K smpl:7M ep:147K epch:29.78 loss:0.008 grdn:0.142 lr:5.9e-06 updt_s:2.353 data_s:1.138 smp/s:37 mem_gb:52.79
271
+ Training: 85%|████████▌ | 51000/60000 [49:40:17<8:42:42, 3.48s/step]INFO 2026-07-19 17:28:56 ot_train.py:646 step:51K smpl:7M ep:147K epch:29.89 loss:0.009 grdn:0.152 lr:5.7e-06 updt_s:2.352 data_s:1.149 smp/s:37 mem_gb:52.71
272
+ Training: 85%|████████▌ | 51200/60000 [49:51:56<8:35:08, 3.51s/step]INFO 2026-07-19 17:40:35 ot_train.py:646 step:51K smpl:7M ep:148K epch:30.01 loss:0.008 grdn:0.145 lr:5.4e-06 updt_s:2.356 data_s:1.140 smp/s:37 mem_gb:52.79
273
+ Training: 86%|████████▌ | 51400/60000 [50:03:35<8:23:28, 3.51s/step]INFO 2026-07-19 17:52:14 ot_train.py:646 step:51K smpl:7M ep:149K epch:30.13 loss:0.008 grdn:0.145 lr:5.2e-06 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
274
+ Training: 86%|████████▌ | 51600/60000 [50:15:14<8:14:26, 3.53s/step]INFO 2026-07-19 18:03:53 ot_train.py:646 step:52K smpl:7M ep:149K epch:30.25 loss:0.008 grdn:0.145 lr:5.0e-06 updt_s:2.353 data_s:1.137 smp/s:37 mem_gb:52.79
275
+ Training: 86%|████████▋ | 51800/60000 [50:26:52<7:56:31, 3.49s/step]INFO 2026-07-19 18:15:31 ot_train.py:646 step:52K smpl:7M ep:150K epch:30.36 loss:0.008 grdn:0.143 lr:4.7e-06 updt_s:2.353 data_s:1.136 smp/s:37 mem_gb:52.79
276
+ Training: 87%|████████▋ | 52000/60000 [50:38:31<7:44:17, 3.48s/step]INFO 2026-07-19 18:27:10 ot_train.py:646 step:52K smpl:7M ep:150K epch:30.48 loss:0.008 grdn:0.145 lr:4.5e-06 updt_s:2.352 data_s:1.144 smp/s:37 mem_gb:52.71
277
+ Training: 87%|████████▋ | 52200/60000 [50:50:10<7:31:46, 3.48s/step]INFO 2026-07-19 18:38:49 ot_train.py:646 step:52K smpl:7M ep:151K epch:30.60 loss:0.008 grdn:0.141 lr:4.3e-06 updt_s:2.357 data_s:1.133 smp/s:37 mem_gb:52.79
278
+ Training: 87%|████████▋ | 52400/60000 [51:01:48<7:22:32, 3.49s/step]INFO 2026-07-19 18:50:27 ot_train.py:646 step:52K smpl:7M ep:151K epch:30.72 loss:0.008 grdn:0.140 lr:4.1e-06 updt_s:2.355 data_s:1.134 smp/s:37 mem_gb:52.79
279
+ Training: 88%|████████▊ | 52600/60000 [51:13:26<7:13:34, 3.52s/step]INFO 2026-07-19 19:02:05 ot_train.py:646 step:53K smpl:7M ep:152K epch:30.83 loss:0.008 grdn:0.143 lr:3.9e-06 updt_s:2.357 data_s:1.131 smp/s:37 mem_gb:52.79
280
+ Training: 88%|████████▊ | 52800/60000 [51:25:05<6:58:35, 3.49s/step]INFO 2026-07-19 19:13:44 ot_train.py:646 step:53K smpl:7M ep:153K epch:30.95 loss:0.008 grdn:0.143 lr:3.7e-06 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
281
+ Training: 88%|████████▊ | 53000/60000 [51:36:46<6:47:17, 3.49s/step]INFO 2026-07-19 19:25:25 ot_train.py:646 step:53K smpl:7M ep:153K epch:31.07 loss:0.008 grdn:0.142 lr:3.5e-06 updt_s:2.364 data_s:1.139 smp/s:37 mem_gb:52.79
282
+ Training: 89%|████████▊ | 53200/60000 [51:48:28<6:34:46, 3.48s/step]INFO 2026-07-19 19:37:07 ot_train.py:646 step:53K smpl:7M ep:154K epch:31.18 loss:0.008 grdn:0.146 lr:3.3e-06 updt_s:2.349 data_s:1.157 smp/s:37 mem_gb:52.71
283
+ Training: 89%|████████▉ | 53400/60000 [52:00:10<6:24:41, 3.50s/step]INFO 2026-07-19 19:48:49 ot_train.py:646 step:53K smpl:7M ep:154K epch:31.30 loss:0.008 grdn:0.144 lr:3.1e-06 updt_s:2.358 data_s:1.150 smp/s:36 mem_gb:52.79
284
+ Training: 89%|████████▉ | 53600/60000 [52:11:49<6:11:16, 3.48s/step]INFO 2026-07-19 20:00:28 ot_train.py:646 step:54K smpl:7M ep:155K epch:31.42 loss:0.008 grdn:0.142 lr:2.9e-06 updt_s:2.353 data_s:1.140 smp/s:37 mem_gb:52.79
285
+ Training: 90%|████████▉ | 53800/60000 [52:23:27<5:59:18, 3.48s/step]INFO 2026-07-19 20:12:06 ot_train.py:646 step:54K smpl:7M ep:155K epch:31.54 loss:0.007 grdn:0.147 lr:2.7e-06 updt_s:2.356 data_s:1.135 smp/s:37 mem_gb:52.79
286
+ Training: 90%|█████████ | 54000/60000 [52:35:08<5:49:45, 3.50s/step]INFO 2026-07-19 20:23:47 ot_train.py:646 step:54K smpl:7M ep:156K epch:31.65 loss:0.008 grdn:0.140 lr:2.6e-06 updt_s:2.365 data_s:1.135 smp/s:37 mem_gb:52.79
287
+ Training: 90%|█████████ | 54200/60000 [52:46:46<5:42:39, 3.54s/step]INFO 2026-07-19 20:35:25 ot_train.py:646 step:54K smpl:7M ep:157K epch:31.77 loss:0.008 grdn:0.141 lr:2.4e-06 updt_s:2.348 data_s:1.141 smp/s:37 mem_gb:52.71
288
+ Training: 91%|█████████ | 54400/60000 [52:58:25<5:23:29, 3.47s/step]INFO 2026-07-19 20:47:04 ot_train.py:646 step:54K smpl:7M ep:157K epch:31.89 loss:0.007 grdn:0.140 lr:2.2e-06 updt_s:2.359 data_s:1.134 smp/s:37 mem_gb:52.79
289
+ Training: 91%|█████████ | 54600/60000 [53:10:04<5:14:13, 3.49s/step]INFO 2026-07-19 20:58:43 ot_train.py:646 step:55K smpl:7M ep:158K epch:32.00 loss:0.008 grdn:0.148 lr:2.1e-06 updt_s:2.360 data_s:1.134 smp/s:37 mem_gb:52.79
290
+ Training: 91%|█████████▏| 54800/60000 [53:21:43<5:02:36, 3.49s/step]INFO 2026-07-19 21:10:22 ot_train.py:646 step:55K smpl:7M ep:158K epch:32.12 loss:0.008 grdn:0.147 lr:1.9e-06 updt_s:2.358 data_s:1.136 smp/s:37 mem_gb:52.79
291
+ Training: 92%|█████████▏| 55000/60000 [53:33:21<4:52:34, 3.51s/step]INFO 2026-07-19 21:22:00 ot_train.py:646 step:55K smpl:7M ep:159K epch:32.24 loss:0.007 grdn:0.146 lr:1.8e-06 updt_s:2.353 data_s:1.133 smp/s:37 mem_gb:52.79
292
+ Training: 92%|█████████▏| 55200/60000 [53:44:58<4:39:57, 3.50s/step]INFO 2026-07-19 21:33:37 ot_train.py:646 step:55K smpl:7M ep:160K epch:32.36 loss:0.007 grdn:0.141 lr:1.7e-06 updt_s:2.352 data_s:1.131 smp/s:37 mem_gb:52.79
293
+ Training: 92%|█████████▏| 55400/60000 [53:56:35<4:26:14, 3.47s/step]INFO 2026-07-19 21:45:14 ot_train.py:646 step:55K smpl:7M ep:160K epch:32.47 loss:0.007 grdn:0.138 lr:1.5e-06 updt_s:2.345 data_s:1.139 smp/s:37 mem_gb:52.71
294
+ Training: 93%|█████████▎| 55600/60000 [54:08:13<4:17:21, 3.51s/step]INFO 2026-07-19 21:56:51 ot_train.py:646 step:56K smpl:7M ep:161K epch:32.59 loss:0.007 grdn:0.143 lr:1.4e-06 updt_s:2.357 data_s:1.130 smp/s:37 mem_gb:52.79
295
+ Training: 93%|█████████▎| 55800/60000 [54:19:50<4:03:56, 3.48s/step]INFO 2026-07-19 22:08:29 ot_train.py:646 step:56K smpl:7M ep:161K epch:32.71 loss:0.007 grdn:0.150 lr:1.3e-06 updt_s:2.356 data_s:1.129 smp/s:37 mem_gb:52.79
296
+ Training: 93%|█████████▎| 56000/60000 [54:31:28<3:51:33, 3.47s/step]INFO 2026-07-19 22:20:07 ot_train.py:646 step:56K smpl:7M ep:162K epch:32.83 loss:0.007 grdn:0.140 lr:1.2e-06 updt_s:2.358 data_s:1.130 smp/s:37 mem_gb:52.79
297
+ Training: 94%|█████████▎| 56200/60000 [54:43:06<3:40:39, 3.48s/step]INFO 2026-07-19 22:31:44 ot_train.py:646 step:56K smpl:7M ep:162K epch:32.94 loss:0.007 grdn:0.142 lr:1.1e-06 updt_s:2.356 data_s:1.130 smp/s:37 mem_gb:52.79
298
+ Training: 94%|█████████▍| 56400/60000 [54:54:44<3:27:34, 3.46s/step]INFO 2026-07-19 22:43:23 ot_train.py:646 step:56K smpl:7M ep:163K epch:33.06 loss:0.007 grdn:0.144 lr:9.5e-07 updt_s:2.360 data_s:1.130 smp/s:37 mem_gb:52.79
299
+ Training: 94%|█████████▍| 56600/60000 [55:06:21<3:17:27, 3.48s/step]INFO 2026-07-19 22:55:00 ot_train.py:646 step:57K smpl:7M ep:164K epch:33.18 loss:0.007 grdn:0.139 lr:8.5e-07 updt_s:2.349 data_s:1.138 smp/s:37 mem_gb:52.71
300
+ Training: 95%|█████████▍| 56800/60000 [55:17:59<3:06:16, 3.49s/step]INFO 2026-07-19 23:06:38 ot_train.py:646 step:57K smpl:7M ep:164K epch:33.29 loss:0.007 grdn:0.143 lr:7.6e-07 updt_s:2.356 data_s:1.128 smp/s:37 mem_gb:52.79
301
+ Training: 95%|█████████▌| 57000/60000 [55:29:36<2:54:47, 3.50s/step]INFO 2026-07-19 23:18:15 ot_train.py:646 step:57K smpl:7M ep:165K epch:33.41 loss:0.007 grdn:0.140 lr:6.7e-07 updt_s:2.356 data_s:1.130 smp/s:37 mem_gb:52.79
302
+ Training: 95%|█████████▌| 57200/60000 [55:41:14<2:42:30, 3.48s/step]INFO 2026-07-19 23:29:53 ot_train.py:646 step:57K smpl:7M ep:165K epch:33.53 loss:0.007 grdn:0.143 lr:5.9e-07 updt_s:2.356 data_s:1.132 smp/s:37 mem_gb:52.79
303
+ Training: 96%|█████████▌| 57400/60000 [55:52:52<2:31:23, 3.49s/step]INFO 2026-07-19 23:41:31 ot_train.py:646 step:57K smpl:7M ep:166K epch:33.65 loss:0.007 grdn:0.142 lr:5.1e-07 updt_s:2.358 data_s:1.131 smp/s:37 mem_gb:52.79
304
+ Training: 96%|█████████▌| 57600/60000 [56:04:31<2:20:02, 3.50s/step]INFO 2026-07-19 23:53:10 ot_train.py:646 step:58K smpl:7M ep:166K epch:33.76 loss:0.007 grdn:0.142 lr:4.4e-07 updt_s:2.352 data_s:1.142 smp/s:37 mem_gb:52.71
305
+ Training: 96%|█████████▋| 57800/60000 [56:16:10<2:07:43, 3.48s/step]INFO 2026-07-20 00:04:49 ot_train.py:646 step:58K smpl:7M ep:167K epch:33.88 loss:0.007 grdn:0.142 lr:3.7e-07 updt_s:2.362 data_s:1.131 smp/s:37 mem_gb:52.79
306
+ Training: 97%|█████████▋| 58000/60000 [56:27:49<1:56:15, 3.49s/step]INFO 2026-07-20 00:16:28 ot_train.py:646 step:58K smpl:7M ep:168K epch:34.00 loss:0.007 grdn:0.139 lr:3.1e-07 updt_s:2.361 data_s:1.132 smp/s:37 mem_gb:52.79
307
+ Training: 97%|█████████▋| 58200/60000 [56:39:28<1:46:44, 3.56s/step]INFO 2026-07-20 00:28:06 ot_train.py:646 step:58K smpl:7M ep:168K epch:34.12 loss:0.007 grdn:0.133 lr:2.5e-07 updt_s:2.360 data_s:1.130 smp/s:37 mem_gb:52.79
308
+ Training: 97%|█████████▋| 58400/60000 [56:51:05<1:32:58, 3.49s/step]INFO 2026-07-20 00:39:44 ot_train.py:646 step:58K smpl:7M ep:169K epch:34.23 loss:0.007 grdn:0.144 lr:2.0e-07 updt_s:2.358 data_s:1.130 smp/s:37 mem_gb:52.79
309
+ Training: 98%|█████████▊| 58600/60000 [57:02:44<1:21:35, 3.50s/step]INFO 2026-07-20 00:51:23 ot_train.py:646 step:59K smpl:8M ep:169K epch:34.35 loss:0.007 grdn:0.140 lr:1.6e-07 updt_s:2.360 data_s:1.133 smp/s:37 mem_gb:52.79
310
+ Training: 98%|█████████▊| 58800/60000 [57:14:23<1:09:39, 3.48s/step]INFO 2026-07-20 01:03:02 ot_train.py:646 step:59K smpl:8M ep:170K epch:34.47 loss:0.007 grdn:0.143 lr:1.2e-07 updt_s:2.351 data_s:1.139 smp/s:37 mem_gb:52.71
311
+ Training: 98%|█████████▊| 59000/60000 [57:26:01<58:25, 3.51s/step]INFO 2026-07-20 01:14:40 ot_train.py:646 step:59K smpl:8M ep:170K epch:34.58 loss:0.007 grdn:0.142 lr:8.4e-08 updt_s:2.360 data_s:1.131 smp/s:37 mem_gb:52.79
312
+ Training: 99%|█████████▊| 59200/60000 [57:37:41<47:08, 3.54s/step]INFO 2026-07-20 01:26:20 ot_train.py:646 step:59K smpl:8M ep:171K epch:34.70 loss:0.007 grdn:0.144 lr:5.7e-08 updt_s:2.364 data_s:1.133 smp/s:37 mem_gb:52.79
313
+ Training: 99%|█████████▉| 59400/60000 [57:49:20<34:51, 3.49s/step]INFO 2026-07-20 01:37:59 ot_train.py:646 step:59K smpl:8M ep:172K epch:34.82 loss:0.007 grdn:0.140 lr:3.4e-08 updt_s:2.361 data_s:1.130 smp/s:37 mem_gb:52.79
314
+ Training: 99%|█████████▉| 59600/60000 [58:00:58<23:15, 3.49s/step]INFO 2026-07-20 01:49:37 ot_train.py:646 step:60K smpl:8M ep:172K epch:34.94 loss:0.007 grdn:0.137 lr:1.8e-08 updt_s:2.361 data_s:1.129 smp/s:37 mem_gb:52.79
315
+ Training: 100%|█████████▉| 59800/60000 [58:12:35<11:36, 3.48s/step]INFO 2026-07-20 02:01:14 ot_train.py:646 step:60K smpl:8M ep:173K epch:35.05 loss:0.007 grdn:0.139 lr:6.5e-09 updt_s:2.355 data_s:1.129 smp/s:37 mem_gb:52.79
316
+ Training: 100%|██████████| 60000/60000 [58:24:13<00:00, 3.51s/step]INFO 2026-07-20 02:12:52 ot_train.py:646 step:60K smpl:8M ep:173K epch:35.17 loss:0.007 grdn:0.147 lr:9.2e-10 updt_s:2.349 data_s:1.139 smp/s:37 mem_gb:52.71
317
+ INFO 2026-07-20 02:12:52 ot_train.py:691 Checkpoint policy after step 60000
318
+ INFO 2026-07-20 02:13:16 in_yaml.py:1245 Saved config, action-space, prompt, and phase provenance in /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline and /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/checkpoints/060000
319
+ Training: 100%|██████████| 60000/60000 [58:24:37<00:00, 3.50s/step]
320
+ INFO 2026-07-20 02:13:16 ot_train.py:777 End of training
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/requirements.txt ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ multiprocess==0.70.19
2
+ nvidia-nvshmem-cu13==3.4.5
3
+ nvidia-cuda-nvrtc==13.0.88
4
+ nvidia-nvjitlink==13.0.88
5
+ numpy==2.2.6
6
+ shellingham==1.5.4
7
+ pip==26.1.2
8
+ zipp==4.1.0
9
+ mypy_extensions==1.1.0
10
+ pillow==12.3.0
11
+ wheel==0.47.0
12
+ cuda-pathfinder==1.5.6
13
+ dm-tree==0.1.10
14
+ hf-xet==1.5.1
15
+ packaging==25.0
16
+ typing-inspect==0.9.0
17
+ nvidia-cuda-runtime==13.0.96
18
+ pyyaml-include==1.4.1
19
+ wrapt==2.2.2
20
+ pytz==2026.2
21
+ Pygments==2.20.0
22
+ urllib3==2.7.0
23
+ yarl==1.24.2
24
+ regex==2026.7.10
25
+ anyio==4.14.2
26
+ timm==1.0.28
27
+ typer==0.26.8
28
+ tokenizers==0.22.2
29
+ lerobot==0.6.0
30
+ annotated-doc==0.0.4
31
+ importlib_metadata==9.0.0
32
+ idna==3.18
33
+ mpmath==1.3.0
34
+ platformdirs==4.10.0
35
+ decord==0.6.0
36
+ transformers==5.5.4
37
+ nvidia-cusparselt-cu13==0.8.0
38
+ accelerate==1.14.0
39
+ torchvision==0.26.0
40
+ torchcodec==0.11.1
41
+ nvidia-cufft==12.0.0.61
42
+ mdurl==0.1.2
43
+ sentry-sdk==2.64.0
44
+ av==15.1.0
45
+ nvidia-cusparse==12.6.3.3
46
+ PyYAML==6.0.3
47
+ python-dateutil==2.9.0.post0
48
+ protobuf==7.35.1
49
+ cmake==4.1.3
50
+ pydantic_core==2.46.4
51
+ requests==2.34.2
52
+ typing-inspection==0.4.2
53
+ nvidia-cufile==1.15.1.6
54
+ MarkupSafe==3.0.3
55
+ wandb==0.27.2
56
+ smmap==5.0.3
57
+ nvidia-cuda-cupti==13.0.85
58
+ charset-normalizer==3.4.9
59
+ h11==0.16.0
60
+ mergedeep==1.3.4
61
+ opencv-python-headless==4.13.0.92
62
+ draccus==0.10.0
63
+ annotated-types==0.7.0
64
+ rich==15.0.0
65
+ networkx==3.6.1
66
+ GitPython==3.1.51
67
+ safetensors==0.8.0
68
+ markdown-it-py==4.2.0
69
+ gitdb==4.0.12
70
+ toml==0.10.2
71
+ filelock==3.29.7
72
+ diffusers==0.35.2
73
+ typing_extensions==4.16.0
74
+ certifi==2026.6.17
75
+ tzdata==2026.3
76
+ nvidia-nccl-cu13==2.28.9
77
+ aiohappyeyeballs==2.7.1
78
+ setuptools==80.10.2
79
+ sympy==1.14.0
80
+ torch==2.11.0
81
+ einops==0.8.2
82
+ nvidia-curand==10.4.0.35
83
+ httpx==0.28.1
84
+ nvidia-cudnn-cu13==9.19.0.56
85
+ frozenlist==1.8.0
86
+ termcolor==3.3.0
87
+ multidict==6.7.1
88
+ httpcore==1.0.9
89
+ psutil==7.2.2
90
+ fsspec==2026.2.0
91
+ dill==0.4.1
92
+ pydantic==2.13.4
93
+ cuda-bindings==13.3.1
94
+ cloudpickle==3.1.2
95
+ aiohttp==3.14.1
96
+ datasets==4.8.5
97
+ gymnasium==1.3.0
98
+ jsonlines==4.0.0
99
+ pyarrow==25.0.0
100
+ attrs==26.1.0
101
+ peft==0.19.1
102
+ nvidia-cublas==13.1.0.3
103
+ propcache==0.5.2
104
+ Jinja2==3.1.6
105
+ aiosignal==1.4.0
106
+ nvidia-nvtx==13.0.85
107
+ triton==3.6.0
108
+ tqdm==4.68.4
109
+ pandas==2.3.3
110
+ click==8.4.2
111
+ Farama-Notifications==0.0.6
112
+ six==1.17.0
113
+ huggingface_hub==1.23.0
114
+ nvidia-cusolver==12.0.4.66
115
+ cuda-toolkit==13.0.2
116
+ xxhash==3.8.1
117
+ absl-py==2.5.0
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-metadata.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-110-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.12.13",
4
+ "startedAt": "2026-07-17T06:48:30.892778Z",
5
+ "args": [
6
+ "--config_path=/tmp/insight_lerobot_cfg_s1elc9y_/baseline.yaml",
7
+ "--batch_size=128"
8
+ ],
9
+ "program": "/home/seonho/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
10
+ "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
11
+ "codePathLocal": "my_scripts/lerobot_train_yaml.py",
12
+ "git": {
13
+ "remote": "git@github.com:paragon7060/clvla.git",
14
+ "commit": "a970d9f98f189faccce13cf2fb61ccf5d55921f3"
15
+ },
16
+ "root": "/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline",
17
+ "host": "worker1",
18
+ "executable": "/home/seonho/miniconda3/envs/lerobot060_groot/bin/python",
19
+ "cpu_count": 32,
20
+ "cpu_count_logical": 64,
21
+ "gpu": "NVIDIA A100 80GB PCIe",
22
+ "gpu_count": 4,
23
+ "disk": {
24
+ "/": {
25
+ "total": "3778130485248",
26
+ "used": "3148893519872"
27
+ }
28
+ },
29
+ "memory": {
30
+ "total": "1081620799488"
31
+ },
32
+ "gpu_nvidia": [
33
+ {
34
+ "name": "NVIDIA A100 80GB PCIe",
35
+ "memoryTotal": "85899345920",
36
+ "cudaCores": 6912,
37
+ "architecture": "Ampere",
38
+ "uuid": "GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9"
39
+ },
40
+ {
41
+ "name": "NVIDIA A100 80GB PCIe",
42
+ "memoryTotal": "85899345920",
43
+ "cudaCores": 6912,
44
+ "architecture": "Ampere",
45
+ "uuid": "GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa"
46
+ },
47
+ {
48
+ "name": "NVIDIA A100 80GB PCIe",
49
+ "memoryTotal": "85899345920",
50
+ "cudaCores": 6912,
51
+ "architecture": "Ampere",
52
+ "uuid": "GPU-ca1c0319-e178-a3c3-ac41-386c2723deac"
53
+ },
54
+ {
55
+ "name": "NVIDIA A100 80GB PCIe",
56
+ "memoryTotal": "85899345920",
57
+ "cudaCores": 6912,
58
+ "architecture": "Ampere",
59
+ "uuid": "GPU-47b6036e-4fde-e399-5515-f73eed752c6d"
60
+ }
61
+ ],
62
+ "cudaVersion": "13.2",
63
+ "writerId": "pbf296fi8oqpaqrnve886ppcvwjoxaoj"
64
+ }
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"train/lr":9.22312585679652e-10,"train/samples_per_s":36.69309955325707,"train/samples":7680000,"train/gpu_mem_gb":52.70832591533661,"train/grad_norm":0.14739508057013154,"_timestamp":1.7844811724342055e+09,"_wandb":{"runtime":210284},"train/steps":60000,"train/episodes":173388.83622525382,"_runtime":210284,"train/epochs":35.17014933575128,"train/update_s":2.3494585001049564,"_step":60000,"train/dataloading_s":1.1389358181296847,"train/loss":0.009968766942620277}
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-core.log ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-07-17T15:48:30.925035413+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpldv3yhws/port-3896698.txt","pid":3896698,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2026-07-17T15:48:30.925459259+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3896698}
3
+ {"time":"2026-07-17T15:48:30.925437749+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3896698-3896942-2058348239/socket","Net":"unix"}}
4
+ {"time":"2026-07-17T15:48:31.110302565+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
+ {"time":"2026-07-17T15:48:31.116186898+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"a3ogbmg4","id":"1(@)"}
6
+ {"time":"2026-07-17T15:48:31.564584695+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"a3ogbmg4","id":"1(@)"}
7
+ {"time":"2026-07-17T15:48:37.272889118+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ijhibshzn6ev"}
8
+ {"time":"2026-07-20T02:13:16.832444619+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
9
+ {"time":"2026-07-20T02:13:16.832520908+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
10
+ {"time":"2026-07-20T02:13:16.832554047+09:00","level":"INFO","msg":"server is shutting down"}
11
+ {"time":"2026-07-20T02:13:16.832569444+09:00","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
12
+ {"time":"2026-07-20T02:13:16.832658051+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
13
+ {"time":"2026-07-20T02:13:16.83267221+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3896698-3896942-2058348239/socket","Net":"unix"}}
14
+ {"time":"2026-07-20T02:13:18.668035591+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
15
+ {"time":"2026-07-20T02:13:18.668080494+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
16
+ {"time":"2026-07-20T02:13:18.668095693+09:00","level":"INFO","msg":"server is closed"}
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
2
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Configure stats pid to 3896698
3
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log
5
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log
6
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():766] calling init triggers
7
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
8
+ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/seonho/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 44, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
9
+ 2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():814] starting backend
10
+ 2026-07-17 15:48:31,110 INFO MainThread:3896698 [wandb_init.py:init():829] sending inform_init request
11
+ 2026-07-17 15:48:31,565 INFO MainThread:3896698 [wandb_init.py:init():834] backend started and connected
12
+ 2026-07-17 15:48:31,568 INFO MainThread:3896698 [wandb_init.py:init():904] updated telemetry
13
+ 2026-07-17 15:48:31,579 INFO MainThread:3896698 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
14
+ 2026-07-17 15:48:32,131 INFO MainThread:3896698 [wandb_init.py:init():972] starting run threads in backend
15
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_console_start():2523] atexit reg
16
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
+ 2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2465] Redirects installed.
19
+ 2026-07-17 15:48:32,267 INFO MainThread:3896698 [wandb_init.py:init():1010] run started, returning control to user process
20
+ 2026-07-20 02:13:16,832 INFO wandb-AsyncioManager-main:3896698 [service_client.py:_forward_responses():122] Reached EOF.
21
+ 2026-07-20 02:13:16,833 INFO wandb-AsyncioManager-main:3896698 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
22
+ 2026-07-20 02:13:16,834 ERROR wandb-AsyncioManager-main:3896698 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
23
+ Traceback (most recent call last):
24
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
25
+ await fn()
26
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
27
+ await asyncio_compat.race(
28
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
29
+ async with open_task_group(race=True) as tg:
30
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
31
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
32
+ await anext(self.gen)
33
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
34
+ await task_group._wait_all(race=race, timeout=exit_timeout)
35
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
36
+ raise exc
37
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
38
+ result = await handle.wait_async(timeout=None)
39
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
40
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
41
+ response = await self._handle.wait_async(timeout=timeout)
42
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
43
+ File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
44
+ raise HandleAbandonedError()
45
+ wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/run-a3ogbmg4.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3b55846504f4d60e38f9eccea1b59427cedb9328f74b91e3c5a48d4504bd7dd
3
+ size 36212873