Whalswp commited on
Commit
25d3612
Β·
verified Β·
1 Parent(s): 6ac8011

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. Abs_6D/Baseline/checkpoints/050000/action_space_manifest.json +141 -0
  2. Abs_6D/Baseline/checkpoints/050000/pretrained_model/action_space_manifest.json +141 -0
  3. Abs_6D/Baseline/checkpoints/050000/pretrained_model/config.json +107 -0
  4. Abs_6D/Baseline/checkpoints/050000/pretrained_model/policy_postprocessor.json +23 -0
  5. Abs_6D/Baseline/checkpoints/050000/pretrained_model/policy_preprocessor.json +78 -0
  6. Abs_6D/Baseline/checkpoints/050000/pretrained_model/prompt_manifest.json +40 -0
  7. Abs_6D/Baseline/checkpoints/050000/pretrained_model/train_config.json +258 -0
  8. Abs_6D/Baseline/checkpoints/050000/resolved_config.yaml +24 -0
  9. Abs_6D/Baseline/checkpoints/050000/source_config.yaml +63 -0
  10. Abs_6D/Baseline/checkpoints/050000/training_state/optimizer_param_groups.json +577 -0
  11. Abs_6D/Baseline/checkpoints/050000/training_state/scheduler_state.json +18 -0
  12. Abs_6D/Baseline/checkpoints/050000/training_state/training_step.json +5 -0
  13. Abs_6D/Baseline/checkpoints/060000/action_space_manifest.json +141 -0
  14. Abs_6D/Baseline/checkpoints/060000/pretrained_model/action_space_manifest.json +141 -0
  15. Abs_6D/Baseline/checkpoints/060000/pretrained_model/config.json +107 -0
  16. Abs_6D/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor.json +23 -0
  17. Abs_6D/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor.json +78 -0
  18. Abs_6D/Baseline/checkpoints/060000/pretrained_model/prompt_manifest.json +40 -0
  19. Abs_6D/Baseline/checkpoints/060000/pretrained_model/train_config.json +258 -0
  20. Abs_6D/Baseline/checkpoints/060000/prompt_manifest.json +40 -0
  21. Abs_6D/Baseline/checkpoints/060000/resolved_config.yaml +24 -0
  22. Abs_6D/Baseline/checkpoints/060000/source_config.yaml +63 -0
  23. Abs_6D/Baseline/checkpoints/060000/training_state/optimizer_param_groups.json +577 -0
  24. Abs_6D/Baseline/checkpoints/060000/training_state/scheduler_state.json +18 -0
  25. Abs_6D/Baseline/checkpoints/060000/training_state/training_step.json +5 -0
  26. Abs_6D/Baseline/wandb/debug-internal.log +0 -0
  27. Abs_6D/Baseline/wandb/debug.log +46 -0
  28. Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/files/output.log +269 -0
  29. Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/files/requirements.txt +117 -0
  30. Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/files/wandb-metadata.json +91 -0
  31. Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug-core.log +8 -0
  32. Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug-internal.log +0 -0
  33. Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug.log +19 -0
  34. Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/config.yaml +351 -0
  35. Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/output.log +74 -0
  36. Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/requirements.txt +117 -0
  37. Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/wandb-metadata.json +92 -0
  38. Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/wandb-summary.json +1 -0
  39. Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug-internal.log +0 -0
  40. Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug.log +46 -0
  41. Abs_6D/Baseline_gpu3_resume.pid +1 -0
  42. Abs_6D/Baseline_gpu3_watcher_status.json +5 -0
  43. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/phase_schedule.json +55 -0
  44. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/pretrained_model/action_space_manifest.json +141 -0
  45. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/pretrained_model/policy_postprocessor.json +23 -0
  46. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/pretrained_model/train_config.json +268 -0
  47. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/resolved_config.yaml +34 -0
  48. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/training_state/training_step.json +5 -0
  49. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/files/output.log +321 -0
  50. Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/logs/debug-internal.log +0 -0
Abs_6D/Baseline/checkpoints/050000/action_space_manifest.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "action_space": "ee_abs_rot6d",
4
+ "source_feature": "action.ee_abs_rot6d",
5
+ "canonical_feature": "action",
6
+ "dtype": "float32",
7
+ "shape": [
8
+ 10
9
+ ],
10
+ "names": [
11
+ "x",
12
+ "y",
13
+ "z",
14
+ "r_col0_x",
15
+ "r_col0_y",
16
+ "r_col0_z",
17
+ "r_col1_x",
18
+ "r_col1_y",
19
+ "r_col1_z",
20
+ "gripper"
21
+ ],
22
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
23
+ "dataset_revision": "v3.0",
24
+ "dataset_total_frames": 218367,
25
+ "dataset_total_episodes": 4930,
26
+ "dataset_fps": 10,
27
+ "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
28
+ "stats": {
29
+ "min": [
30
+ -0.21784618496894836,
31
+ -0.4474811851978302,
32
+ 0.1702737659215927,
33
+ -0.9999986290931702,
34
+ -0.9999991655349731,
35
+ -0.9999999403953552,
36
+ -0.9999990463256836,
37
+ -0.9999998211860657,
38
+ -0.9999988079071045,
39
+ 0.0
40
+ ],
41
+ "max": [
42
+ 0.6283698678016663,
43
+ 0.4420371949672699,
44
+ 1.051027774810791,
45
+ 0.9999999403953552,
46
+ 0.9999995231628418,
47
+ 1.0,
48
+ 0.9999995231628418,
49
+ 1.0,
50
+ 0.9999996423721313,
51
+ 0.03999999910593033
52
+ ],
53
+ "mean": [
54
+ 0.4040228037735655,
55
+ -0.05247618696989652,
56
+ 0.6993459771218912,
57
+ 0.07011111991957117,
58
+ -0.03520558904460197,
59
+ 0.14470242846953205,
60
+ 0.05491166606739672,
61
+ -0.19073058893378805,
62
+ -0.06988117661871228,
63
+ 0.021167300266524632
64
+ ],
65
+ "std": [
66
+ 0.11861927913357073,
67
+ 0.11318943557281885,
68
+ 0.17849160646987516,
69
+ 0.4812528791886511,
70
+ 0.7487196258736938,
71
+ 0.4251126804030219,
72
+ 0.47797229807769137,
73
+ 0.6267695346407992,
74
+ 0.5782954306815554,
75
+ 0.01996590431050817
76
+ ],
77
+ "count": [
78
+ 218367
79
+ ],
80
+ "q01": [
81
+ 0.23988881827972064,
82
+ -0.10387461883034811,
83
+ 0.6297243923476871,
84
+ -0.43622841955353464,
85
+ -0.6885486527617446,
86
+ -0.14103555388039019,
87
+ -0.4327210063987595,
88
+ -0.9304790915716145,
89
+ -0.3465462258490417,
90
+ -1.000000013351432e-10
91
+ ],
92
+ "q10": [
93
+ 0.3042306436755216,
94
+ -0.09609991620622624,
95
+ 0.6394063072781145,
96
+ -0.3351306352998669,
97
+ -0.5751963015002924,
98
+ -0.08490621283307866,
99
+ -0.33849701676224275,
100
+ -0.8340557712529717,
101
+ -0.29760290790884514,
102
+ -1.000000013351432e-10
103
+ ],
104
+ "q50": [
105
+ 0.4160225263528291,
106
+ -0.06267459152989589,
107
+ 0.6811777491060801,
108
+ 0.0543536902230377,
109
+ -0.14012251058819217,
110
+ 0.04772394280004727,
111
+ 0.03838119138716219,
112
+ -0.14865169094195635,
113
+ -0.2073069401172425,
114
+ 0.020772594934870346
115
+ ],
116
+ "q90": [
117
+ 0.46641008755679864,
118
+ 0.020233143101890064,
119
+ 0.8095135405259374,
120
+ 0.48172401592474906,
121
+ 0.7398737043070319,
122
+ 0.5970841215636613,
123
+ 0.4674752875999555,
124
+ 0.36281754777672676,
125
+ 0.5033361883834496,
126
+ 0.039998362213352394
127
+ ],
128
+ "q99": [
129
+ 0.47320723171849244,
130
+ 0.04114262476135705,
131
+ 0.8467967259270389,
132
+ 0.5867781409980988,
133
+ 0.8529584859257228,
134
+ 0.7953936054341789,
135
+ 0.5768269858732034,
136
+ 0.49353349323389695,
137
+ 0.7168914440776504,
138
+ 0.03999983541667277
139
+ ]
140
+ }
141
+ }
Abs_6D/Baseline/checkpoints/050000/pretrained_model/action_space_manifest.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "action_space": "ee_abs_rot6d",
4
+ "source_feature": "action.ee_abs_rot6d",
5
+ "canonical_feature": "action",
6
+ "dtype": "float32",
7
+ "shape": [
8
+ 10
9
+ ],
10
+ "names": [
11
+ "x",
12
+ "y",
13
+ "z",
14
+ "r_col0_x",
15
+ "r_col0_y",
16
+ "r_col0_z",
17
+ "r_col1_x",
18
+ "r_col1_y",
19
+ "r_col1_z",
20
+ "gripper"
21
+ ],
22
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
23
+ "dataset_revision": "v3.0",
24
+ "dataset_total_frames": 218367,
25
+ "dataset_total_episodes": 4930,
26
+ "dataset_fps": 10,
27
+ "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
28
+ "stats": {
29
+ "min": [
30
+ -0.21784618496894836,
31
+ -0.4474811851978302,
32
+ 0.1702737659215927,
33
+ -0.9999986290931702,
34
+ -0.9999991655349731,
35
+ -0.9999999403953552,
36
+ -0.9999990463256836,
37
+ -0.9999998211860657,
38
+ -0.9999988079071045,
39
+ 0.0
40
+ ],
41
+ "max": [
42
+ 0.6283698678016663,
43
+ 0.4420371949672699,
44
+ 1.051027774810791,
45
+ 0.9999999403953552,
46
+ 0.9999995231628418,
47
+ 1.0,
48
+ 0.9999995231628418,
49
+ 1.0,
50
+ 0.9999996423721313,
51
+ 0.03999999910593033
52
+ ],
53
+ "mean": [
54
+ 0.4040228037735655,
55
+ -0.05247618696989652,
56
+ 0.6993459771218912,
57
+ 0.07011111991957117,
58
+ -0.03520558904460197,
59
+ 0.14470242846953205,
60
+ 0.05491166606739672,
61
+ -0.19073058893378805,
62
+ -0.06988117661871228,
63
+ 0.021167300266524632
64
+ ],
65
+ "std": [
66
+ 0.11861927913357073,
67
+ 0.11318943557281885,
68
+ 0.17849160646987516,
69
+ 0.4812528791886511,
70
+ 0.7487196258736938,
71
+ 0.4251126804030219,
72
+ 0.47797229807769137,
73
+ 0.6267695346407992,
74
+ 0.5782954306815554,
75
+ 0.01996590431050817
76
+ ],
77
+ "count": [
78
+ 218367
79
+ ],
80
+ "q01": [
81
+ 0.23988881827972064,
82
+ -0.10387461883034811,
83
+ 0.6297243923476871,
84
+ -0.43622841955353464,
85
+ -0.6885486527617446,
86
+ -0.14103555388039019,
87
+ -0.4327210063987595,
88
+ -0.9304790915716145,
89
+ -0.3465462258490417,
90
+ -1.000000013351432e-10
91
+ ],
92
+ "q10": [
93
+ 0.3042306436755216,
94
+ -0.09609991620622624,
95
+ 0.6394063072781145,
96
+ -0.3351306352998669,
97
+ -0.5751963015002924,
98
+ -0.08490621283307866,
99
+ -0.33849701676224275,
100
+ -0.8340557712529717,
101
+ -0.29760290790884514,
102
+ -1.000000013351432e-10
103
+ ],
104
+ "q50": [
105
+ 0.4160225263528291,
106
+ -0.06267459152989589,
107
+ 0.6811777491060801,
108
+ 0.0543536902230377,
109
+ -0.14012251058819217,
110
+ 0.04772394280004727,
111
+ 0.03838119138716219,
112
+ -0.14865169094195635,
113
+ -0.2073069401172425,
114
+ 0.020772594934870346
115
+ ],
116
+ "q90": [
117
+ 0.46641008755679864,
118
+ 0.020233143101890064,
119
+ 0.8095135405259374,
120
+ 0.48172401592474906,
121
+ 0.7398737043070319,
122
+ 0.5970841215636613,
123
+ 0.4674752875999555,
124
+ 0.36281754777672676,
125
+ 0.5033361883834496,
126
+ 0.039998362213352394
127
+ ],
128
+ "q99": [
129
+ 0.47320723171849244,
130
+ 0.04114262476135705,
131
+ 0.8467967259270389,
132
+ 0.5867781409980988,
133
+ 0.8529584859257228,
134
+ 0.7953936054341789,
135
+ 0.5768269858732034,
136
+ 0.49353349323389695,
137
+ 0.7168914440776504,
138
+ 0.03999983541667277
139
+ ]
140
+ }
141
+ }
Abs_6D/Baseline/checkpoints/050000/pretrained_model/config.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "groot",
3
+ "n_obs_steps": 1,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 16
9
+ ]
10
+ },
11
+ "observation.images.wrist": {
12
+ "type": "VISUAL",
13
+ "shape": [
14
+ 3,
15
+ 224,
16
+ 224
17
+ ]
18
+ },
19
+ "observation.images.right_shoulder": {
20
+ "type": "VISUAL",
21
+ "shape": [
22
+ 3,
23
+ 224,
24
+ 224
25
+ ]
26
+ },
27
+ "observation.images.guide": {
28
+ "type": "VISUAL",
29
+ "shape": [
30
+ 3,
31
+ 224,
32
+ 224
33
+ ]
34
+ }
35
+ },
36
+ "output_features": {
37
+ "action": {
38
+ "type": "ACTION",
39
+ "shape": [
40
+ 10
41
+ ]
42
+ }
43
+ },
44
+ "device": "cuda",
45
+ "use_amp": false,
46
+ "use_peft": false,
47
+ "push_to_hub": false,
48
+ "repo_id": null,
49
+ "private": null,
50
+ "tags": null,
51
+ "license": null,
52
+ "pretrained_path": null,
53
+ "pretrained_revision": null,
54
+ "chunk_size": 16,
55
+ "n_action_steps": 16,
56
+ "max_state_dim": 132,
57
+ "max_action_dim": 132,
58
+ "normalization_mapping": {
59
+ "VISUAL": "IDENTITY",
60
+ "STATE": "IDENTITY",
61
+ "ACTION": "IDENTITY"
62
+ },
63
+ "base_model_path": "nvidia/GR00T-N1.7-3B",
64
+ "action_decode_transform": null,
65
+ "embodiment_tag": "new_embodiment",
66
+ "tune_llm": false,
67
+ "tune_visual": false,
68
+ "tune_projector": true,
69
+ "tune_diffusion_model": true,
70
+ "tune_vlln": true,
71
+ "tune_top_llm_layers": 0,
72
+ "num_inference_timesteps": null,
73
+ "rtc_ramp_rate": null,
74
+ "use_flash_attention": false,
75
+ "use_relative_actions": false,
76
+ "relative_exclude_joints": [],
77
+ "optimizer_lr": 0.0001,
78
+ "optimizer_betas": [
79
+ 0.9,
80
+ 0.999
81
+ ],
82
+ "optimizer_eps": 1e-08,
83
+ "optimizer_weight_decay": 1e-05,
84
+ "warmup_ratio": 0.05,
85
+ "use_bf16": true,
86
+ "model_params_fp32": true,
87
+ "image_size": [
88
+ 256,
89
+ 256
90
+ ],
91
+ "tokenizer_assets_repo": null,
92
+ "lora_rank": 0,
93
+ "lora_alpha": 16,
94
+ "lora_dropout": 0.1,
95
+ "lora_full_model": false,
96
+ "video_backend": "decord",
97
+ "balance_dataset_weights": true,
98
+ "balance_trajectory_weights": true,
99
+ "dataset_paths": null,
100
+ "output_dir": "./tmp/gr00t",
101
+ "save_steps": 1000,
102
+ "max_steps": 10000,
103
+ "batch_size": 32,
104
+ "dataloader_num_workers": 8,
105
+ "report_to": "wandb",
106
+ "resume": false
107
+ }
Abs_6D/Baseline/checkpoints/050000/pretrained_model/policy_postprocessor.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "groot_action_unpack_unnormalize_v2",
6
+ "config": {
7
+ "env_action_dim": 10,
8
+ "normalize_min_max": true,
9
+ "clip_normalized_action": true,
10
+ "libero_gripper_action": false,
11
+ "libero_gripper_binarize": true
12
+ },
13
+ "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
14
+ },
15
+ {
16
+ "registry_name": "device_processor",
17
+ "config": {
18
+ "device": "cpu",
19
+ "float_dtype": null
20
+ }
21
+ }
22
+ ]
23
+ }
Abs_6D/Baseline/checkpoints/050000/pretrained_model/policy_preprocessor.json ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {}
8
+ }
9
+ },
10
+ {
11
+ "registry_name": "to_batch_processor",
12
+ "config": {}
13
+ },
14
+ {
15
+ "registry_name": "groot_n1_7_pack_inputs_v1",
16
+ "config": {
17
+ "state_horizon": 1,
18
+ "action_horizon": 16,
19
+ "valid_action_horizon": 16,
20
+ "video_horizon": null,
21
+ "max_state_dim": 132,
22
+ "max_action_dim": 132,
23
+ "language_key": "task",
24
+ "formalize_language": true,
25
+ "embodiment_tag": "new_embodiment",
26
+ "embodiment_mapping": {
27
+ "oxe_droid_relative_eef_relative_joint": 24,
28
+ "xdof_relative_eef_relative_joint": 27,
29
+ "xdof_relative_eef_relative_joint_subtask": 27,
30
+ "real_g1_relative_eef_relative_joints": 25,
31
+ "real_r1_pro_sharpa_relative_eef": 26,
32
+ "real_r1_pro_sharpa_relative_eef_human": 26,
33
+ "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
34
+ "real_r1_pro_sharpa_relative_eef_mecka": 26,
35
+ "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
36
+ "simpler_env_google": 0,
37
+ "simpler_env_widowx": 1,
38
+ "libero_sim": 2,
39
+ "new_embodiment": 10
40
+ },
41
+ "normalize_min_max": true,
42
+ "state_dropout_prob": 0.0,
43
+ "clip_outliers": true,
44
+ "use_percentiles": false,
45
+ "video_modality_keys": null,
46
+ "raw_stats": null,
47
+ "modality_config": null
48
+ },
49
+ "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors"
50
+ },
51
+ {
52
+ "registry_name": "groot_n1_7_vlm_encode_v1",
53
+ "config": {
54
+ "model_name": "nvidia/Cosmos-Reason2-2B",
55
+ "image_crop_size": [
56
+ 230,
57
+ 230
58
+ ],
59
+ "image_target_size": [
60
+ 256,
61
+ 256
62
+ ],
63
+ "shortest_image_edge": null,
64
+ "crop_fraction": null,
65
+ "use_albumentations": false,
66
+ "letter_box_transform": false,
67
+ "device": "cuda"
68
+ }
69
+ },
70
+ {
71
+ "registry_name": "device_processor",
72
+ "config": {
73
+ "device": "cuda",
74
+ "float_dtype": null
75
+ }
76
+ }
77
+ ]
78
+ }
Abs_6D/Baseline/checkpoints/050000/pretrained_model/prompt_manifest.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "prompt_set": "detailed",
4
+ "source_file": "task_prompts.json",
5
+ "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
6
+ "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
7
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
8
+ "dataset_revision": "v3.0",
9
+ "task_count": 13,
10
+ "task_codes": [
11
+ "5f",
12
+ "5g",
13
+ "1ext",
14
+ "3b",
15
+ "5b",
16
+ "3a",
17
+ "3c",
18
+ "5h",
19
+ "5d",
20
+ "3d",
21
+ "5e",
22
+ "5a",
23
+ "5c"
24
+ ],
25
+ "prompts": {
26
+ "5f": "Close the bottle in clockwise direction.",
27
+ "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
28
+ "1ext": "Find the arrow guide and open the indicated drawer.",
29
+ "3b": "Open the door, rotate counter-clockwise and push.",
30
+ "5b": "Open the bottle in counter-clockwise direction.",
31
+ "3a": "Open the door, rotate clockwise and push.",
32
+ "3c": "Open the door, rotate clockwise and pull.",
33
+ "5h": "Close the bottle in counter-clockwise direction.",
34
+ "5d": "Close the bottle in clockwise direction.",
35
+ "3d": "Open the door, rotate counter-clockwise and pull.",
36
+ "5e": "Close the bottle in counter-clockwise direction.",
37
+ "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
38
+ "5c": "Open the bottle in clockwise direction."
39
+ }
40
+ }
Abs_6D/Baseline/checkpoints/050000/pretrained_model/train_config.json ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
4
+ "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "tfs": {
11
+ "brightness": {
12
+ "weight": 1.0,
13
+ "type": "ColorJitter",
14
+ "kwargs": {
15
+ "brightness": [
16
+ 0.8,
17
+ 1.2
18
+ ]
19
+ }
20
+ },
21
+ "contrast": {
22
+ "weight": 1.0,
23
+ "type": "ColorJitter",
24
+ "kwargs": {
25
+ "contrast": [
26
+ 0.8,
27
+ 1.2
28
+ ]
29
+ }
30
+ },
31
+ "saturation": {
32
+ "weight": 1.0,
33
+ "type": "ColorJitter",
34
+ "kwargs": {
35
+ "saturation": [
36
+ 0.5,
37
+ 1.5
38
+ ]
39
+ }
40
+ },
41
+ "hue": {
42
+ "weight": 1.0,
43
+ "type": "ColorJitter",
44
+ "kwargs": {
45
+ "hue": [
46
+ -0.05,
47
+ 0.05
48
+ ]
49
+ }
50
+ },
51
+ "sharpness": {
52
+ "weight": 1.0,
53
+ "type": "SharpnessJitter",
54
+ "kwargs": {
55
+ "sharpness": [
56
+ 0.5,
57
+ 1.5
58
+ ]
59
+ }
60
+ },
61
+ "affine": {
62
+ "weight": 1.0,
63
+ "type": "RandomAffine",
64
+ "kwargs": {
65
+ "degrees": [
66
+ -5.0,
67
+ 5.0
68
+ ],
69
+ "translate": [
70
+ 0.05,
71
+ 0.05
72
+ ]
73
+ }
74
+ }
75
+ }
76
+ },
77
+ "revision": null,
78
+ "use_imagenet_stats": true,
79
+ "video_backend": "torchcodec",
80
+ "return_uint8": false,
81
+ "depth_output_unit": "mm",
82
+ "streaming": false,
83
+ "eval_split": 0.0
84
+ },
85
+ "env": null,
86
+ "policy": {
87
+ "type": "groot",
88
+ "n_obs_steps": 1,
89
+ "input_features": {
90
+ "observation.state": {
91
+ "type": "STATE",
92
+ "shape": [
93
+ 16
94
+ ]
95
+ },
96
+ "observation.images.wrist": {
97
+ "type": "VISUAL",
98
+ "shape": [
99
+ 3,
100
+ 224,
101
+ 224
102
+ ]
103
+ },
104
+ "observation.images.right_shoulder": {
105
+ "type": "VISUAL",
106
+ "shape": [
107
+ 3,
108
+ 224,
109
+ 224
110
+ ]
111
+ },
112
+ "observation.images.guide": {
113
+ "type": "VISUAL",
114
+ "shape": [
115
+ 3,
116
+ 224,
117
+ 224
118
+ ]
119
+ }
120
+ },
121
+ "output_features": {
122
+ "action": {
123
+ "type": "ACTION",
124
+ "shape": [
125
+ 10
126
+ ]
127
+ }
128
+ },
129
+ "device": "cuda",
130
+ "use_amp": false,
131
+ "use_peft": false,
132
+ "push_to_hub": false,
133
+ "repo_id": null,
134
+ "private": null,
135
+ "tags": null,
136
+ "license": null,
137
+ "pretrained_path": null,
138
+ "pretrained_revision": null,
139
+ "chunk_size": 16,
140
+ "n_action_steps": 16,
141
+ "max_state_dim": 132,
142
+ "max_action_dim": 132,
143
+ "normalization_mapping": {
144
+ "VISUAL": "IDENTITY",
145
+ "STATE": "IDENTITY",
146
+ "ACTION": "IDENTITY"
147
+ },
148
+ "base_model_path": "nvidia/GR00T-N1.7-3B",
149
+ "action_decode_transform": null,
150
+ "embodiment_tag": "new_embodiment",
151
+ "tune_llm": false,
152
+ "tune_visual": false,
153
+ "tune_projector": true,
154
+ "tune_diffusion_model": true,
155
+ "tune_vlln": true,
156
+ "tune_top_llm_layers": 0,
157
+ "num_inference_timesteps": null,
158
+ "rtc_ramp_rate": null,
159
+ "use_flash_attention": false,
160
+ "use_relative_actions": false,
161
+ "relative_exclude_joints": [],
162
+ "optimizer_lr": 0.0001,
163
+ "optimizer_betas": [
164
+ 0.9,
165
+ 0.999
166
+ ],
167
+ "optimizer_eps": 1e-08,
168
+ "optimizer_weight_decay": 1e-05,
169
+ "warmup_ratio": 0.05,
170
+ "use_bf16": true,
171
+ "model_params_fp32": true,
172
+ "image_size": [
173
+ 256,
174
+ 256
175
+ ],
176
+ "tokenizer_assets_repo": null,
177
+ "lora_rank": 0,
178
+ "lora_alpha": 16,
179
+ "lora_dropout": 0.1,
180
+ "lora_full_model": false,
181
+ "video_backend": "decord",
182
+ "balance_dataset_weights": true,
183
+ "balance_trajectory_weights": true,
184
+ "dataset_paths": null,
185
+ "output_dir": "./tmp/gr00t",
186
+ "save_steps": 1000,
187
+ "max_steps": 10000,
188
+ "batch_size": 32,
189
+ "dataloader_num_workers": 8,
190
+ "report_to": "wandb",
191
+ "resume": false
192
+ },
193
+ "reward_model": null,
194
+ "output_dir": "/home/ext_minje/groot_insight/Abs_6D/Baseline",
195
+ "job_name": "INSIGHT_6D_baseline",
196
+ "resume": false,
197
+ "seed": 42,
198
+ "cudnn_deterministic": false,
199
+ "num_workers": 4,
200
+ "batch_size": 64,
201
+ "prefetch_factor": 4,
202
+ "persistent_workers": true,
203
+ "steps": 60000,
204
+ "env_eval_freq": 20000,
205
+ "log_freq": 200,
206
+ "eval_steps": 0,
207
+ "max_eval_samples": 0,
208
+ "tolerance_s": 0.0001,
209
+ "save_checkpoint": true,
210
+ "save_freq": 20000,
211
+ "use_policy_training_preset": true,
212
+ "optimizer": {
213
+ "type": "adamw",
214
+ "lr": 0.0001,
215
+ "weight_decay": 1e-05,
216
+ "grad_clip_norm": 1.0,
217
+ "betas": [
218
+ 0.9,
219
+ 0.999
220
+ ],
221
+ "eps": 1e-08
222
+ },
223
+ "scheduler": {
224
+ "type": "diffuser",
225
+ "num_warmup_steps": 500,
226
+ "name": "cosine"
227
+ },
228
+ "eval": {
229
+ "n_episodes": 50,
230
+ "batch_size": 50,
231
+ "use_async_envs": true,
232
+ "recording": false,
233
+ "recording_repo_id": null,
234
+ "recording_private": false
235
+ },
236
+ "wandb": {
237
+ "enable": true,
238
+ "disable_artifact": true,
239
+ "project": "lerobot",
240
+ "entity": null,
241
+ "notes": null,
242
+ "run_id": "1jmjb68j",
243
+ "mode": null,
244
+ "add_tags": true
245
+ },
246
+ "peft": null,
247
+ "job": {
248
+ "target": null,
249
+ "image": "huggingface/lerobot-gpu:latest",
250
+ "timeout": "2d",
251
+ "detach": false,
252
+ "tags": []
253
+ },
254
+ "save_checkpoint_to_hub": false,
255
+ "sample_weighting": null,
256
+ "rename_map": {},
257
+ "checkpoint_path": null
258
+ }
Abs_6D/Baseline/checkpoints/050000/resolved_config.yaml ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
3
+ root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
4
+ policy:
5
+ type: groot
6
+ device: cuda
7
+ chunk_size: 16
8
+ n_action_steps: 16
9
+ push_to_hub: false
10
+ tune_llm: false
11
+ tune_visual: false
12
+ tune_top_llm_layers: 0
13
+ tune_projector: true
14
+ tune_diffusion_model: true
15
+ tune_vlln: true
16
+ seed: 42
17
+ batch_size: 64
18
+ steps: 60000
19
+ log_freq: 200
20
+ output_dir: /home/ext_minje/groot_insight/Abs_6D/Baseline
21
+ job_name: INSIGHT_6D_baseline
22
+ wandb:
23
+ enable: true
24
+ disable_artifact: true
Abs_6D/Baseline/checkpoints/050000/source_config.yaml ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # INSIGHT GR00T N1.7 training config
2
+ # INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
3
+ insight:
4
+ # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
5
+ action_space: ee_abs_rot6d
6
+ # Options: coarse | detailed
7
+ prompt_set: detailed
8
+ save_steps: ["50K"]
9
+
10
+ dataset:
11
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
12
+ root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
13
+
14
+ policy:
15
+ type: groot
16
+ device: cuda
17
+ chunk_size: 16
18
+ n_action_steps: 16
19
+ push_to_hub: false
20
+ # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
21
+ # VLM
22
+ tune_llm: false
23
+ tune_visual: false
24
+ tune_top_llm_layers: 0
25
+ # Action Expert
26
+ tune_projector: true
27
+ tune_diffusion_model: true
28
+ tune_vlln: true
29
+
30
+
31
+ seed: 42
32
+ batch_size: 64
33
+ steps: 60000
34
+ log_freq: 200
35
+ output_dir: ${HOME}/groot_insight/Abs_6D/Baseline
36
+ job_name: INSIGHT_6D_baseline
37
+
38
+ wandb:
39
+ enable: true
40
+ disable_artifact: true
41
+
42
+ # Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
43
+ #
44
+ # dataset:
45
+ # eval_split: 0.0
46
+ # image_transforms:
47
+ # enable: false # default; set true only for train-image augmentation.
48
+ #
49
+ # policy:
50
+ # base_model_path: nvidia/GR00T-N1.7-3B
51
+ # embodiment_tag: new_embodiment
52
+ # use_relative_actions: false
53
+ # use_bf16: true
54
+ # repo_id: Whalswp/INSIGHT # unused while push_to_hub is false
55
+ #
56
+ # save_checkpoint: true
57
+ # use_policy_training_preset: true
58
+ # eval_steps: 0
59
+ # env_eval_freq: 0
60
+ #
61
+ # insight.prompt_set options:
62
+ # coarse: High-level task text that requires following the visual guide.
63
+ # detailed: Direction- and interaction-specific text, including rotation and push/pull details.
Abs_6D/Baseline/checkpoints/050000/training_state/optimizer_param_groups.json ADDED
@@ -0,0 +1,577 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "lr": 6.809148352279182e-06,
4
+ "betas": [
5
+ 0.9,
6
+ 0.999
7
+ ],
8
+ "eps": 1e-08,
9
+ "weight_decay": 1e-05,
10
+ "amsgrad": false,
11
+ "maximize": false,
12
+ "foreach": null,
13
+ "capturable": false,
14
+ "differentiable": false,
15
+ "fused": null,
16
+ "decoupled_weight_decay": true,
17
+ "initial_lr": 0.0001,
18
+ "params": [
19
+ 0,
20
+ 1,
21
+ 2,
22
+ 3,
23
+ 4,
24
+ 5,
25
+ 6,
26
+ 7,
27
+ 8,
28
+ 9,
29
+ 10,
30
+ 11,
31
+ 12,
32
+ 13,
33
+ 14,
34
+ 15,
35
+ 16,
36
+ 17,
37
+ 18,
38
+ 19,
39
+ 20,
40
+ 21,
41
+ 22,
42
+ 23,
43
+ 24,
44
+ 25,
45
+ 26,
46
+ 27,
47
+ 28,
48
+ 29,
49
+ 30,
50
+ 31,
51
+ 32,
52
+ 33,
53
+ 34,
54
+ 35,
55
+ 36,
56
+ 37,
57
+ 38,
58
+ 39,
59
+ 40,
60
+ 41,
61
+ 42,
62
+ 43,
63
+ 44,
64
+ 45,
65
+ 46,
66
+ 47,
67
+ 48,
68
+ 49,
69
+ 50,
70
+ 51,
71
+ 52,
72
+ 53,
73
+ 54,
74
+ 55,
75
+ 56,
76
+ 57,
77
+ 58,
78
+ 59,
79
+ 60,
80
+ 61,
81
+ 62,
82
+ 63,
83
+ 64,
84
+ 65,
85
+ 66,
86
+ 67,
87
+ 68,
88
+ 69,
89
+ 70,
90
+ 71,
91
+ 72,
92
+ 73,
93
+ 74,
94
+ 75,
95
+ 76,
96
+ 77,
97
+ 78,
98
+ 79,
99
+ 80,
100
+ 81,
101
+ 82,
102
+ 83,
103
+ 84,
104
+ 85,
105
+ 86,
106
+ 87,
107
+ 88,
108
+ 89,
109
+ 90,
110
+ 91,
111
+ 92,
112
+ 93,
113
+ 94,
114
+ 95,
115
+ 96,
116
+ 97,
117
+ 98,
118
+ 99,
119
+ 100,
120
+ 101,
121
+ 102,
122
+ 103,
123
+ 104,
124
+ 105,
125
+ 106,
126
+ 107,
127
+ 108,
128
+ 109,
129
+ 110,
130
+ 111,
131
+ 112,
132
+ 113,
133
+ 114,
134
+ 115,
135
+ 116,
136
+ 117,
137
+ 118,
138
+ 119,
139
+ 120,
140
+ 121,
141
+ 122,
142
+ 123,
143
+ 124,
144
+ 125,
145
+ 126,
146
+ 127,
147
+ 128,
148
+ 129,
149
+ 130,
150
+ 131,
151
+ 132,
152
+ 133,
153
+ 134,
154
+ 135,
155
+ 136,
156
+ 137,
157
+ 138,
158
+ 139,
159
+ 140,
160
+ 141,
161
+ 142,
162
+ 143,
163
+ 144,
164
+ 145,
165
+ 146,
166
+ 147,
167
+ 148,
168
+ 149,
169
+ 150,
170
+ 151,
171
+ 152,
172
+ 153,
173
+ 154,
174
+ 155,
175
+ 156,
176
+ 157,
177
+ 158,
178
+ 159,
179
+ 160,
180
+ 161,
181
+ 162,
182
+ 163,
183
+ 164,
184
+ 165,
185
+ 166,
186
+ 167,
187
+ 168,
188
+ 169,
189
+ 170,
190
+ 171,
191
+ 172,
192
+ 173,
193
+ 174,
194
+ 175,
195
+ 176,
196
+ 177,
197
+ 178,
198
+ 179,
199
+ 180,
200
+ 181,
201
+ 182,
202
+ 183,
203
+ 184,
204
+ 185,
205
+ 186,
206
+ 187,
207
+ 188,
208
+ 189,
209
+ 190,
210
+ 191,
211
+ 192,
212
+ 193,
213
+ 194,
214
+ 195,
215
+ 196,
216
+ 197,
217
+ 198,
218
+ 199,
219
+ 200,
220
+ 201,
221
+ 202,
222
+ 203,
223
+ 204,
224
+ 205,
225
+ 206,
226
+ 207,
227
+ 208,
228
+ 209,
229
+ 210,
230
+ 211,
231
+ 212,
232
+ 213,
233
+ 214,
234
+ 215,
235
+ 216,
236
+ 217,
237
+ 218,
238
+ 219,
239
+ 220,
240
+ 221,
241
+ 222,
242
+ 223,
243
+ 224,
244
+ 225,
245
+ 226,
246
+ 227,
247
+ 228,
248
+ 229,
249
+ 230,
250
+ 231,
251
+ 232,
252
+ 233,
253
+ 234,
254
+ 235,
255
+ 236,
256
+ 237,
257
+ 238,
258
+ 239,
259
+ 240,
260
+ 241,
261
+ 242,
262
+ 243,
263
+ 244,
264
+ 245,
265
+ 246,
266
+ 247,
267
+ 248,
268
+ 249,
269
+ 250,
270
+ 251,
271
+ 252,
272
+ 253,
273
+ 254,
274
+ 255,
275
+ 256,
276
+ 257,
277
+ 258,
278
+ 259,
279
+ 260,
280
+ 261,
281
+ 262,
282
+ 263,
283
+ 264,
284
+ 265,
285
+ 266
286
+ ]
287
+ },
288
+ {
289
+ "weight_decay": 0.0,
290
+ "lr": 6.809148352279182e-06,
291
+ "betas": [
292
+ 0.9,
293
+ 0.999
294
+ ],
295
+ "eps": 1e-08,
296
+ "amsgrad": false,
297
+ "maximize": false,
298
+ "foreach": null,
299
+ "capturable": false,
300
+ "differentiable": false,
301
+ "fused": null,
302
+ "decoupled_weight_decay": true,
303
+ "initial_lr": 0.0001,
304
+ "params": [
305
+ 267,
306
+ 268,
307
+ 269,
308
+ 270,
309
+ 271,
310
+ 272,
311
+ 273,
312
+ 274,
313
+ 275,
314
+ 276,
315
+ 277,
316
+ 278,
317
+ 279,
318
+ 280,
319
+ 281,
320
+ 282,
321
+ 283,
322
+ 284,
323
+ 285,
324
+ 286,
325
+ 287,
326
+ 288,
327
+ 289,
328
+ 290,
329
+ 291,
330
+ 292,
331
+ 293,
332
+ 294,
333
+ 295,
334
+ 296,
335
+ 297,
336
+ 298,
337
+ 299,
338
+ 300,
339
+ 301,
340
+ 302,
341
+ 303,
342
+ 304,
343
+ 305,
344
+ 306,
345
+ 307,
346
+ 308,
347
+ 309,
348
+ 310,
349
+ 311,
350
+ 312,
351
+ 313,
352
+ 314,
353
+ 315,
354
+ 316,
355
+ 317,
356
+ 318,
357
+ 319,
358
+ 320,
359
+ 321,
360
+ 322,
361
+ 323,
362
+ 324,
363
+ 325,
364
+ 326,
365
+ 327,
366
+ 328,
367
+ 329,
368
+ 330,
369
+ 331,
370
+ 332,
371
+ 333,
372
+ 334,
373
+ 335,
374
+ 336,
375
+ 337,
376
+ 338,
377
+ 339,
378
+ 340,
379
+ 341,
380
+ 342,
381
+ 343,
382
+ 344,
383
+ 345,
384
+ 346,
385
+ 347,
386
+ 348,
387
+ 349,
388
+ 350,
389
+ 351,
390
+ 352,
391
+ 353,
392
+ 354,
393
+ 355,
394
+ 356,
395
+ 357,
396
+ 358,
397
+ 359,
398
+ 360,
399
+ 361,
400
+ 362,
401
+ 363,
402
+ 364,
403
+ 365,
404
+ 366,
405
+ 367,
406
+ 368,
407
+ 369,
408
+ 370,
409
+ 371,
410
+ 372,
411
+ 373,
412
+ 374,
413
+ 375,
414
+ 376,
415
+ 377,
416
+ 378,
417
+ 379,
418
+ 380,
419
+ 381,
420
+ 382,
421
+ 383,
422
+ 384,
423
+ 385,
424
+ 386,
425
+ 387,
426
+ 388,
427
+ 389,
428
+ 390,
429
+ 391,
430
+ 392,
431
+ 393,
432
+ 394,
433
+ 395,
434
+ 396,
435
+ 397,
436
+ 398,
437
+ 399,
438
+ 400,
439
+ 401,
440
+ 402,
441
+ 403,
442
+ 404,
443
+ 405,
444
+ 406,
445
+ 407,
446
+ 408,
447
+ 409,
448
+ 410,
449
+ 411,
450
+ 412,
451
+ 413,
452
+ 414,
453
+ 415,
454
+ 416,
455
+ 417,
456
+ 418,
457
+ 419,
458
+ 420,
459
+ 421,
460
+ 422,
461
+ 423,
462
+ 424,
463
+ 425,
464
+ 426,
465
+ 427,
466
+ 428,
467
+ 429,
468
+ 430,
469
+ 431,
470
+ 432,
471
+ 433,
472
+ 434,
473
+ 435,
474
+ 436,
475
+ 437,
476
+ 438,
477
+ 439,
478
+ 440,
479
+ 441,
480
+ 442,
481
+ 443,
482
+ 444,
483
+ 445,
484
+ 446,
485
+ 447,
486
+ 448,
487
+ 449,
488
+ 450,
489
+ 451,
490
+ 452,
491
+ 453,
492
+ 454,
493
+ 455,
494
+ 456,
495
+ 457,
496
+ 458,
497
+ 459,
498
+ 460,
499
+ 461,
500
+ 462,
501
+ 463,
502
+ 464,
503
+ 465,
504
+ 466,
505
+ 467,
506
+ 468,
507
+ 469,
508
+ 470,
509
+ 471,
510
+ 472,
511
+ 473,
512
+ 474,
513
+ 475,
514
+ 476,
515
+ 477,
516
+ 478,
517
+ 479,
518
+ 480,
519
+ 481,
520
+ 482,
521
+ 483,
522
+ 484,
523
+ 485,
524
+ 486,
525
+ 487,
526
+ 488,
527
+ 489,
528
+ 490,
529
+ 491,
530
+ 492,
531
+ 493,
532
+ 494,
533
+ 495,
534
+ 496,
535
+ 497,
536
+ 498,
537
+ 499,
538
+ 500,
539
+ 501,
540
+ 502,
541
+ 503,
542
+ 504,
543
+ 505,
544
+ 506,
545
+ 507,
546
+ 508,
547
+ 509,
548
+ 510,
549
+ 511,
550
+ 512,
551
+ 513,
552
+ 514,
553
+ 515,
554
+ 516,
555
+ 517,
556
+ 518,
557
+ 519,
558
+ 520,
559
+ 521,
560
+ 522,
561
+ 523,
562
+ 524,
563
+ 525,
564
+ 526,
565
+ 527,
566
+ 528,
567
+ 529,
568
+ 530,
569
+ 531,
570
+ 532,
571
+ 533,
572
+ 534,
573
+ 535,
574
+ 536
575
+ ]
576
+ }
577
+ ]
Abs_6D/Baseline/checkpoints/050000/training_state/scheduler_state.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_lrs": [
3
+ 0.0001,
4
+ 0.0001
5
+ ],
6
+ "last_epoch": 50000,
7
+ "_step_count": 50001,
8
+ "_is_initial": false,
9
+ "_get_lr_called_within_step": false,
10
+ "_last_lr": [
11
+ 6.809148352279182e-06,
12
+ 6.809148352279182e-06
13
+ ],
14
+ "lr_lambdas": [
15
+ null,
16
+ null
17
+ ]
18
+ }
Abs_6D/Baseline/checkpoints/050000/training_state/training_step.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "step": 50000,
3
+ "num_processes": 1,
4
+ "batch_size": 64
5
+ }
Abs_6D/Baseline/checkpoints/060000/action_space_manifest.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "action_space": "ee_abs_rot6d",
4
+ "source_feature": "action.ee_abs_rot6d",
5
+ "canonical_feature": "action",
6
+ "dtype": "float32",
7
+ "shape": [
8
+ 10
9
+ ],
10
+ "names": [
11
+ "x",
12
+ "y",
13
+ "z",
14
+ "r_col0_x",
15
+ "r_col0_y",
16
+ "r_col0_z",
17
+ "r_col1_x",
18
+ "r_col1_y",
19
+ "r_col1_z",
20
+ "gripper"
21
+ ],
22
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
23
+ "dataset_revision": "v3.0",
24
+ "dataset_total_frames": 218367,
25
+ "dataset_total_episodes": 4930,
26
+ "dataset_fps": 10,
27
+ "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
28
+ "stats": {
29
+ "min": [
30
+ -0.21784618496894836,
31
+ -0.4474811851978302,
32
+ 0.1702737659215927,
33
+ -0.9999986290931702,
34
+ -0.9999991655349731,
35
+ -0.9999999403953552,
36
+ -0.9999990463256836,
37
+ -0.9999998211860657,
38
+ -0.9999988079071045,
39
+ 0.0
40
+ ],
41
+ "max": [
42
+ 0.6283698678016663,
43
+ 0.4420371949672699,
44
+ 1.051027774810791,
45
+ 0.9999999403953552,
46
+ 0.9999995231628418,
47
+ 1.0,
48
+ 0.9999995231628418,
49
+ 1.0,
50
+ 0.9999996423721313,
51
+ 0.03999999910593033
52
+ ],
53
+ "mean": [
54
+ 0.4040228037735655,
55
+ -0.05247618696989652,
56
+ 0.6993459771218912,
57
+ 0.07011111991957117,
58
+ -0.03520558904460197,
59
+ 0.14470242846953205,
60
+ 0.05491166606739672,
61
+ -0.19073058893378805,
62
+ -0.06988117661871228,
63
+ 0.021167300266524632
64
+ ],
65
+ "std": [
66
+ 0.11861927913357073,
67
+ 0.11318943557281885,
68
+ 0.17849160646987516,
69
+ 0.4812528791886511,
70
+ 0.7487196258736938,
71
+ 0.4251126804030219,
72
+ 0.47797229807769137,
73
+ 0.6267695346407992,
74
+ 0.5782954306815554,
75
+ 0.01996590431050817
76
+ ],
77
+ "count": [
78
+ 218367
79
+ ],
80
+ "q01": [
81
+ 0.23988881827972064,
82
+ -0.10387461883034811,
83
+ 0.6297243923476871,
84
+ -0.43622841955353464,
85
+ -0.6885486527617446,
86
+ -0.14103555388039019,
87
+ -0.4327210063987595,
88
+ -0.9304790915716145,
89
+ -0.3465462258490417,
90
+ -1.000000013351432e-10
91
+ ],
92
+ "q10": [
93
+ 0.3042306436755216,
94
+ -0.09609991620622624,
95
+ 0.6394063072781145,
96
+ -0.3351306352998669,
97
+ -0.5751963015002924,
98
+ -0.08490621283307866,
99
+ -0.33849701676224275,
100
+ -0.8340557712529717,
101
+ -0.29760290790884514,
102
+ -1.000000013351432e-10
103
+ ],
104
+ "q50": [
105
+ 0.4160225263528291,
106
+ -0.06267459152989589,
107
+ 0.6811777491060801,
108
+ 0.0543536902230377,
109
+ -0.14012251058819217,
110
+ 0.04772394280004727,
111
+ 0.03838119138716219,
112
+ -0.14865169094195635,
113
+ -0.2073069401172425,
114
+ 0.020772594934870346
115
+ ],
116
+ "q90": [
117
+ 0.46641008755679864,
118
+ 0.020233143101890064,
119
+ 0.8095135405259374,
120
+ 0.48172401592474906,
121
+ 0.7398737043070319,
122
+ 0.5970841215636613,
123
+ 0.4674752875999555,
124
+ 0.36281754777672676,
125
+ 0.5033361883834496,
126
+ 0.039998362213352394
127
+ ],
128
+ "q99": [
129
+ 0.47320723171849244,
130
+ 0.04114262476135705,
131
+ 0.8467967259270389,
132
+ 0.5867781409980988,
133
+ 0.8529584859257228,
134
+ 0.7953936054341789,
135
+ 0.5768269858732034,
136
+ 0.49353349323389695,
137
+ 0.7168914440776504,
138
+ 0.03999983541667277
139
+ ]
140
+ }
141
+ }
Abs_6D/Baseline/checkpoints/060000/pretrained_model/action_space_manifest.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "action_space": "ee_abs_rot6d",
4
+ "source_feature": "action.ee_abs_rot6d",
5
+ "canonical_feature": "action",
6
+ "dtype": "float32",
7
+ "shape": [
8
+ 10
9
+ ],
10
+ "names": [
11
+ "x",
12
+ "y",
13
+ "z",
14
+ "r_col0_x",
15
+ "r_col0_y",
16
+ "r_col0_z",
17
+ "r_col1_x",
18
+ "r_col1_y",
19
+ "r_col1_z",
20
+ "gripper"
21
+ ],
22
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
23
+ "dataset_revision": "v3.0",
24
+ "dataset_total_frames": 218367,
25
+ "dataset_total_episodes": 4930,
26
+ "dataset_fps": 10,
27
+ "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
28
+ "stats": {
29
+ "min": [
30
+ -0.21784618496894836,
31
+ -0.4474811851978302,
32
+ 0.1702737659215927,
33
+ -0.9999986290931702,
34
+ -0.9999991655349731,
35
+ -0.9999999403953552,
36
+ -0.9999990463256836,
37
+ -0.9999998211860657,
38
+ -0.9999988079071045,
39
+ 0.0
40
+ ],
41
+ "max": [
42
+ 0.6283698678016663,
43
+ 0.4420371949672699,
44
+ 1.051027774810791,
45
+ 0.9999999403953552,
46
+ 0.9999995231628418,
47
+ 1.0,
48
+ 0.9999995231628418,
49
+ 1.0,
50
+ 0.9999996423721313,
51
+ 0.03999999910593033
52
+ ],
53
+ "mean": [
54
+ 0.4040228037735655,
55
+ -0.05247618696989652,
56
+ 0.6993459771218912,
57
+ 0.07011111991957117,
58
+ -0.03520558904460197,
59
+ 0.14470242846953205,
60
+ 0.05491166606739672,
61
+ -0.19073058893378805,
62
+ -0.06988117661871228,
63
+ 0.021167300266524632
64
+ ],
65
+ "std": [
66
+ 0.11861927913357073,
67
+ 0.11318943557281885,
68
+ 0.17849160646987516,
69
+ 0.4812528791886511,
70
+ 0.7487196258736938,
71
+ 0.4251126804030219,
72
+ 0.47797229807769137,
73
+ 0.6267695346407992,
74
+ 0.5782954306815554,
75
+ 0.01996590431050817
76
+ ],
77
+ "count": [
78
+ 218367
79
+ ],
80
+ "q01": [
81
+ 0.23988881827972064,
82
+ -0.10387461883034811,
83
+ 0.6297243923476871,
84
+ -0.43622841955353464,
85
+ -0.6885486527617446,
86
+ -0.14103555388039019,
87
+ -0.4327210063987595,
88
+ -0.9304790915716145,
89
+ -0.3465462258490417,
90
+ -1.000000013351432e-10
91
+ ],
92
+ "q10": [
93
+ 0.3042306436755216,
94
+ -0.09609991620622624,
95
+ 0.6394063072781145,
96
+ -0.3351306352998669,
97
+ -0.5751963015002924,
98
+ -0.08490621283307866,
99
+ -0.33849701676224275,
100
+ -0.8340557712529717,
101
+ -0.29760290790884514,
102
+ -1.000000013351432e-10
103
+ ],
104
+ "q50": [
105
+ 0.4160225263528291,
106
+ -0.06267459152989589,
107
+ 0.6811777491060801,
108
+ 0.0543536902230377,
109
+ -0.14012251058819217,
110
+ 0.04772394280004727,
111
+ 0.03838119138716219,
112
+ -0.14865169094195635,
113
+ -0.2073069401172425,
114
+ 0.020772594934870346
115
+ ],
116
+ "q90": [
117
+ 0.46641008755679864,
118
+ 0.020233143101890064,
119
+ 0.8095135405259374,
120
+ 0.48172401592474906,
121
+ 0.7398737043070319,
122
+ 0.5970841215636613,
123
+ 0.4674752875999555,
124
+ 0.36281754777672676,
125
+ 0.5033361883834496,
126
+ 0.039998362213352394
127
+ ],
128
+ "q99": [
129
+ 0.47320723171849244,
130
+ 0.04114262476135705,
131
+ 0.8467967259270389,
132
+ 0.5867781409980988,
133
+ 0.8529584859257228,
134
+ 0.7953936054341789,
135
+ 0.5768269858732034,
136
+ 0.49353349323389695,
137
+ 0.7168914440776504,
138
+ 0.03999983541667277
139
+ ]
140
+ }
141
+ }
Abs_6D/Baseline/checkpoints/060000/pretrained_model/config.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "groot",
3
+ "n_obs_steps": 1,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 16
9
+ ]
10
+ },
11
+ "observation.images.wrist": {
12
+ "type": "VISUAL",
13
+ "shape": [
14
+ 3,
15
+ 224,
16
+ 224
17
+ ]
18
+ },
19
+ "observation.images.right_shoulder": {
20
+ "type": "VISUAL",
21
+ "shape": [
22
+ 3,
23
+ 224,
24
+ 224
25
+ ]
26
+ },
27
+ "observation.images.guide": {
28
+ "type": "VISUAL",
29
+ "shape": [
30
+ 3,
31
+ 224,
32
+ 224
33
+ ]
34
+ }
35
+ },
36
+ "output_features": {
37
+ "action": {
38
+ "type": "ACTION",
39
+ "shape": [
40
+ 10
41
+ ]
42
+ }
43
+ },
44
+ "device": "cuda",
45
+ "use_amp": false,
46
+ "use_peft": false,
47
+ "push_to_hub": false,
48
+ "repo_id": null,
49
+ "private": null,
50
+ "tags": null,
51
+ "license": null,
52
+ "pretrained_path": "/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model",
53
+ "pretrained_revision": null,
54
+ "chunk_size": 16,
55
+ "n_action_steps": 16,
56
+ "max_state_dim": 132,
57
+ "max_action_dim": 132,
58
+ "normalization_mapping": {
59
+ "VISUAL": "IDENTITY",
60
+ "STATE": "IDENTITY",
61
+ "ACTION": "IDENTITY"
62
+ },
63
+ "base_model_path": "nvidia/GR00T-N1.7-3B",
64
+ "action_decode_transform": null,
65
+ "embodiment_tag": "new_embodiment",
66
+ "tune_llm": false,
67
+ "tune_visual": false,
68
+ "tune_projector": true,
69
+ "tune_diffusion_model": true,
70
+ "tune_vlln": true,
71
+ "tune_top_llm_layers": 0,
72
+ "num_inference_timesteps": null,
73
+ "rtc_ramp_rate": null,
74
+ "use_flash_attention": false,
75
+ "use_relative_actions": false,
76
+ "relative_exclude_joints": [],
77
+ "optimizer_lr": 0.0001,
78
+ "optimizer_betas": [
79
+ 0.9,
80
+ 0.999
81
+ ],
82
+ "optimizer_eps": 1e-08,
83
+ "optimizer_weight_decay": 1e-05,
84
+ "warmup_ratio": 0.05,
85
+ "use_bf16": true,
86
+ "model_params_fp32": true,
87
+ "image_size": [
88
+ 256,
89
+ 256
90
+ ],
91
+ "tokenizer_assets_repo": null,
92
+ "lora_rank": 0,
93
+ "lora_alpha": 16,
94
+ "lora_dropout": 0.1,
95
+ "lora_full_model": false,
96
+ "video_backend": "decord",
97
+ "balance_dataset_weights": true,
98
+ "balance_trajectory_weights": true,
99
+ "dataset_paths": null,
100
+ "output_dir": "./tmp/gr00t",
101
+ "save_steps": 1000,
102
+ "max_steps": 10000,
103
+ "batch_size": 32,
104
+ "dataloader_num_workers": 8,
105
+ "report_to": "wandb",
106
+ "resume": false
107
+ }
Abs_6D/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "groot_action_unpack_unnormalize_v2",
6
+ "config": {
7
+ "env_action_dim": 10,
8
+ "normalize_min_max": true,
9
+ "clip_normalized_action": true,
10
+ "libero_gripper_action": false,
11
+ "libero_gripper_binarize": true
12
+ },
13
+ "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
14
+ },
15
+ {
16
+ "registry_name": "device_processor",
17
+ "config": {
18
+ "device": "cpu",
19
+ "float_dtype": null
20
+ }
21
+ }
22
+ ]
23
+ }
Abs_6D/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor.json ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {}
8
+ }
9
+ },
10
+ {
11
+ "registry_name": "to_batch_processor",
12
+ "config": {}
13
+ },
14
+ {
15
+ "registry_name": "groot_n1_7_pack_inputs_v1",
16
+ "config": {
17
+ "state_horizon": 1,
18
+ "action_horizon": 16,
19
+ "valid_action_horizon": 16,
20
+ "video_horizon": null,
21
+ "max_state_dim": 132,
22
+ "max_action_dim": 132,
23
+ "language_key": "task",
24
+ "formalize_language": true,
25
+ "embodiment_tag": "new_embodiment",
26
+ "embodiment_mapping": {
27
+ "oxe_droid_relative_eef_relative_joint": 24,
28
+ "xdof_relative_eef_relative_joint": 27,
29
+ "xdof_relative_eef_relative_joint_subtask": 27,
30
+ "real_g1_relative_eef_relative_joints": 25,
31
+ "real_r1_pro_sharpa_relative_eef": 26,
32
+ "real_r1_pro_sharpa_relative_eef_human": 26,
33
+ "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
34
+ "real_r1_pro_sharpa_relative_eef_mecka": 26,
35
+ "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
36
+ "simpler_env_google": 0,
37
+ "simpler_env_widowx": 1,
38
+ "libero_sim": 2,
39
+ "new_embodiment": 10
40
+ },
41
+ "normalize_min_max": true,
42
+ "state_dropout_prob": 0.0,
43
+ "clip_outliers": true,
44
+ "use_percentiles": false,
45
+ "video_modality_keys": null,
46
+ "raw_stats": null,
47
+ "modality_config": null
48
+ },
49
+ "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors"
50
+ },
51
+ {
52
+ "registry_name": "groot_n1_7_vlm_encode_v1",
53
+ "config": {
54
+ "model_name": "nvidia/Cosmos-Reason2-2B",
55
+ "image_crop_size": [
56
+ 230,
57
+ 230
58
+ ],
59
+ "image_target_size": [
60
+ 256,
61
+ 256
62
+ ],
63
+ "shortest_image_edge": null,
64
+ "crop_fraction": null,
65
+ "use_albumentations": false,
66
+ "letter_box_transform": false,
67
+ "device": "cuda"
68
+ }
69
+ },
70
+ {
71
+ "registry_name": "device_processor",
72
+ "config": {
73
+ "device": "cuda",
74
+ "float_dtype": null
75
+ }
76
+ }
77
+ ]
78
+ }
Abs_6D/Baseline/checkpoints/060000/pretrained_model/prompt_manifest.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "prompt_set": "detailed",
4
+ "source_file": "task_prompts.json",
5
+ "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
6
+ "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
7
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
8
+ "dataset_revision": "v3.0",
9
+ "task_count": 13,
10
+ "task_codes": [
11
+ "5f",
12
+ "5g",
13
+ "1ext",
14
+ "3b",
15
+ "5b",
16
+ "3a",
17
+ "3c",
18
+ "5h",
19
+ "5d",
20
+ "3d",
21
+ "5e",
22
+ "5a",
23
+ "5c"
24
+ ],
25
+ "prompts": {
26
+ "5f": "Close the bottle in clockwise direction.",
27
+ "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
28
+ "1ext": "Find the arrow guide and open the indicated drawer.",
29
+ "3b": "Open the door, rotate counter-clockwise and push.",
30
+ "5b": "Open the bottle in counter-clockwise direction.",
31
+ "3a": "Open the door, rotate clockwise and push.",
32
+ "3c": "Open the door, rotate clockwise and pull.",
33
+ "5h": "Close the bottle in counter-clockwise direction.",
34
+ "5d": "Close the bottle in clockwise direction.",
35
+ "3d": "Open the door, rotate counter-clockwise and pull.",
36
+ "5e": "Close the bottle in counter-clockwise direction.",
37
+ "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
38
+ "5c": "Open the bottle in clockwise direction."
39
+ }
40
+ }
Abs_6D/Baseline/checkpoints/060000/pretrained_model/train_config.json ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
4
+ "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "tfs": {
11
+ "brightness": {
12
+ "weight": 1.0,
13
+ "type": "ColorJitter",
14
+ "kwargs": {
15
+ "brightness": [
16
+ 0.8,
17
+ 1.2
18
+ ]
19
+ }
20
+ },
21
+ "contrast": {
22
+ "weight": 1.0,
23
+ "type": "ColorJitter",
24
+ "kwargs": {
25
+ "contrast": [
26
+ 0.8,
27
+ 1.2
28
+ ]
29
+ }
30
+ },
31
+ "saturation": {
32
+ "weight": 1.0,
33
+ "type": "ColorJitter",
34
+ "kwargs": {
35
+ "saturation": [
36
+ 0.5,
37
+ 1.5
38
+ ]
39
+ }
40
+ },
41
+ "hue": {
42
+ "weight": 1.0,
43
+ "type": "ColorJitter",
44
+ "kwargs": {
45
+ "hue": [
46
+ -0.05,
47
+ 0.05
48
+ ]
49
+ }
50
+ },
51
+ "sharpness": {
52
+ "weight": 1.0,
53
+ "type": "SharpnessJitter",
54
+ "kwargs": {
55
+ "sharpness": [
56
+ 0.5,
57
+ 1.5
58
+ ]
59
+ }
60
+ },
61
+ "affine": {
62
+ "weight": 1.0,
63
+ "type": "RandomAffine",
64
+ "kwargs": {
65
+ "degrees": [
66
+ -5.0,
67
+ 5.0
68
+ ],
69
+ "translate": [
70
+ 0.05,
71
+ 0.05
72
+ ]
73
+ }
74
+ }
75
+ }
76
+ },
77
+ "revision": null,
78
+ "use_imagenet_stats": true,
79
+ "video_backend": "torchcodec",
80
+ "return_uint8": false,
81
+ "depth_output_unit": "mm",
82
+ "streaming": false,
83
+ "eval_split": 0.0
84
+ },
85
+ "env": null,
86
+ "policy": {
87
+ "type": "groot",
88
+ "n_obs_steps": 1,
89
+ "input_features": {
90
+ "observation.state": {
91
+ "type": "STATE",
92
+ "shape": [
93
+ 16
94
+ ]
95
+ },
96
+ "observation.images.wrist": {
97
+ "type": "VISUAL",
98
+ "shape": [
99
+ 3,
100
+ 224,
101
+ 224
102
+ ]
103
+ },
104
+ "observation.images.right_shoulder": {
105
+ "type": "VISUAL",
106
+ "shape": [
107
+ 3,
108
+ 224,
109
+ 224
110
+ ]
111
+ },
112
+ "observation.images.guide": {
113
+ "type": "VISUAL",
114
+ "shape": [
115
+ 3,
116
+ 224,
117
+ 224
118
+ ]
119
+ }
120
+ },
121
+ "output_features": {
122
+ "action": {
123
+ "type": "ACTION",
124
+ "shape": [
125
+ 10
126
+ ]
127
+ }
128
+ },
129
+ "device": "cuda",
130
+ "use_amp": false,
131
+ "use_peft": false,
132
+ "push_to_hub": false,
133
+ "repo_id": null,
134
+ "private": null,
135
+ "tags": null,
136
+ "license": null,
137
+ "pretrained_path": "/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model",
138
+ "pretrained_revision": null,
139
+ "chunk_size": 16,
140
+ "n_action_steps": 16,
141
+ "max_state_dim": 132,
142
+ "max_action_dim": 132,
143
+ "normalization_mapping": {
144
+ "VISUAL": "IDENTITY",
145
+ "STATE": "IDENTITY",
146
+ "ACTION": "IDENTITY"
147
+ },
148
+ "base_model_path": "nvidia/GR00T-N1.7-3B",
149
+ "action_decode_transform": null,
150
+ "embodiment_tag": "new_embodiment",
151
+ "tune_llm": false,
152
+ "tune_visual": false,
153
+ "tune_projector": true,
154
+ "tune_diffusion_model": true,
155
+ "tune_vlln": true,
156
+ "tune_top_llm_layers": 0,
157
+ "num_inference_timesteps": null,
158
+ "rtc_ramp_rate": null,
159
+ "use_flash_attention": false,
160
+ "use_relative_actions": false,
161
+ "relative_exclude_joints": [],
162
+ "optimizer_lr": 0.0001,
163
+ "optimizer_betas": [
164
+ 0.9,
165
+ 0.999
166
+ ],
167
+ "optimizer_eps": 1e-08,
168
+ "optimizer_weight_decay": 1e-05,
169
+ "warmup_ratio": 0.05,
170
+ "use_bf16": true,
171
+ "model_params_fp32": true,
172
+ "image_size": [
173
+ 256,
174
+ 256
175
+ ],
176
+ "tokenizer_assets_repo": null,
177
+ "lora_rank": 0,
178
+ "lora_alpha": 16,
179
+ "lora_dropout": 0.1,
180
+ "lora_full_model": false,
181
+ "video_backend": "decord",
182
+ "balance_dataset_weights": true,
183
+ "balance_trajectory_weights": true,
184
+ "dataset_paths": null,
185
+ "output_dir": "./tmp/gr00t",
186
+ "save_steps": 1000,
187
+ "max_steps": 10000,
188
+ "batch_size": 32,
189
+ "dataloader_num_workers": 8,
190
+ "report_to": "wandb",
191
+ "resume": false
192
+ },
193
+ "reward_model": null,
194
+ "output_dir": "/home/ext_minje/groot_insight/Abs_6D/Baseline",
195
+ "job_name": "INSIGHT_6D_baseline",
196
+ "resume": true,
197
+ "seed": 42,
198
+ "cudnn_deterministic": false,
199
+ "num_workers": 4,
200
+ "batch_size": 64,
201
+ "prefetch_factor": 4,
202
+ "persistent_workers": true,
203
+ "steps": 60000,
204
+ "env_eval_freq": 20000,
205
+ "log_freq": 200,
206
+ "eval_steps": 0,
207
+ "max_eval_samples": 0,
208
+ "tolerance_s": 0.0001,
209
+ "save_checkpoint": true,
210
+ "save_freq": 20000,
211
+ "use_policy_training_preset": true,
212
+ "optimizer": {
213
+ "type": "adamw",
214
+ "lr": 0.0001,
215
+ "weight_decay": 1e-05,
216
+ "grad_clip_norm": 1.0,
217
+ "betas": [
218
+ 0.9,
219
+ 0.999
220
+ ],
221
+ "eps": 1e-08
222
+ },
223
+ "scheduler": {
224
+ "type": "diffuser",
225
+ "num_warmup_steps": 500,
226
+ "name": "cosine"
227
+ },
228
+ "eval": {
229
+ "n_episodes": 50,
230
+ "batch_size": 50,
231
+ "use_async_envs": true,
232
+ "recording": false,
233
+ "recording_repo_id": null,
234
+ "recording_private": false
235
+ },
236
+ "wandb": {
237
+ "enable": true,
238
+ "disable_artifact": true,
239
+ "project": "lerobot",
240
+ "entity": null,
241
+ "notes": null,
242
+ "run_id": "1jmjb68j",
243
+ "mode": null,
244
+ "add_tags": true
245
+ },
246
+ "peft": null,
247
+ "job": {
248
+ "target": null,
249
+ "image": "huggingface/lerobot-gpu:latest",
250
+ "timeout": "2d",
251
+ "detach": false,
252
+ "tags": []
253
+ },
254
+ "save_checkpoint_to_hub": false,
255
+ "sample_weighting": null,
256
+ "rename_map": {},
257
+ "checkpoint_path": "/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000"
258
+ }
Abs_6D/Baseline/checkpoints/060000/prompt_manifest.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "prompt_set": "detailed",
4
+ "source_file": "task_prompts.json",
5
+ "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
6
+ "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
7
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
8
+ "dataset_revision": "v3.0",
9
+ "task_count": 13,
10
+ "task_codes": [
11
+ "5f",
12
+ "5g",
13
+ "1ext",
14
+ "3b",
15
+ "5b",
16
+ "3a",
17
+ "3c",
18
+ "5h",
19
+ "5d",
20
+ "3d",
21
+ "5e",
22
+ "5a",
23
+ "5c"
24
+ ],
25
+ "prompts": {
26
+ "5f": "Close the bottle in clockwise direction.",
27
+ "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
28
+ "1ext": "Find the arrow guide and open the indicated drawer.",
29
+ "3b": "Open the door, rotate counter-clockwise and push.",
30
+ "5b": "Open the bottle in counter-clockwise direction.",
31
+ "3a": "Open the door, rotate clockwise and push.",
32
+ "3c": "Open the door, rotate clockwise and pull.",
33
+ "5h": "Close the bottle in counter-clockwise direction.",
34
+ "5d": "Close the bottle in clockwise direction.",
35
+ "3d": "Open the door, rotate counter-clockwise and pull.",
36
+ "5e": "Close the bottle in counter-clockwise direction.",
37
+ "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
38
+ "5c": "Open the bottle in clockwise direction."
39
+ }
40
+ }
Abs_6D/Baseline/checkpoints/060000/resolved_config.yaml ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
3
+ root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
4
+ policy:
5
+ type: groot
6
+ device: cuda
7
+ chunk_size: 16
8
+ n_action_steps: 16
9
+ push_to_hub: false
10
+ tune_llm: false
11
+ tune_visual: false
12
+ tune_top_llm_layers: 0
13
+ tune_projector: true
14
+ tune_diffusion_model: true
15
+ tune_vlln: true
16
+ seed: 42
17
+ batch_size: 64
18
+ steps: 60000
19
+ log_freq: 200
20
+ output_dir: /home/ext_minje/groot_insight/Abs_6D/Baseline
21
+ job_name: INSIGHT_6D_baseline
22
+ wandb:
23
+ enable: true
24
+ disable_artifact: true
Abs_6D/Baseline/checkpoints/060000/source_config.yaml ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # INSIGHT GR00T N1.7 training config
2
+ # INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
3
+ insight:
4
+ # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
5
+ action_space: ee_abs_rot6d
6
+ # Options: coarse | detailed
7
+ prompt_set: detailed
8
+ save_steps: ["50K"]
9
+
10
+ dataset:
11
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
12
+ root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
13
+
14
+ policy:
15
+ type: groot
16
+ device: cuda
17
+ chunk_size: 16
18
+ n_action_steps: 16
19
+ push_to_hub: false
20
+ # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
21
+ # VLM
22
+ tune_llm: false
23
+ tune_visual: false
24
+ tune_top_llm_layers: 0
25
+ # Action Expert
26
+ tune_projector: true
27
+ tune_diffusion_model: true
28
+ tune_vlln: true
29
+
30
+
31
+ seed: 42
32
+ batch_size: 64
33
+ steps: 60000
34
+ log_freq: 200
35
+ output_dir: ${HOME}/groot_insight/Abs_6D/Baseline
36
+ job_name: INSIGHT_6D_baseline
37
+
38
+ wandb:
39
+ enable: true
40
+ disable_artifact: true
41
+
42
+ # Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
43
+ #
44
+ # dataset:
45
+ # eval_split: 0.0
46
+ # image_transforms:
47
+ # enable: false # default; set true only for train-image augmentation.
48
+ #
49
+ # policy:
50
+ # base_model_path: nvidia/GR00T-N1.7-3B
51
+ # embodiment_tag: new_embodiment
52
+ # use_relative_actions: false
53
+ # use_bf16: true
54
+ # repo_id: Whalswp/INSIGHT # unused while push_to_hub is false
55
+ #
56
+ # save_checkpoint: true
57
+ # use_policy_training_preset: true
58
+ # eval_steps: 0
59
+ # env_eval_freq: 0
60
+ #
61
+ # insight.prompt_set options:
62
+ # coarse: High-level task text that requires following the visual guide.
63
+ # detailed: Direction- and interaction-specific text, including rotation and push/pull details.
Abs_6D/Baseline/checkpoints/060000/training_state/optimizer_param_groups.json ADDED
@@ -0,0 +1,577 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "lr": 0.0,
4
+ "betas": [
5
+ 0.9,
6
+ 0.999
7
+ ],
8
+ "eps": 1e-08,
9
+ "weight_decay": 1e-05,
10
+ "amsgrad": false,
11
+ "maximize": false,
12
+ "foreach": null,
13
+ "capturable": false,
14
+ "differentiable": false,
15
+ "fused": null,
16
+ "decoupled_weight_decay": true,
17
+ "initial_lr": 0.0001,
18
+ "params": [
19
+ 0,
20
+ 1,
21
+ 2,
22
+ 3,
23
+ 4,
24
+ 5,
25
+ 6,
26
+ 7,
27
+ 8,
28
+ 9,
29
+ 10,
30
+ 11,
31
+ 12,
32
+ 13,
33
+ 14,
34
+ 15,
35
+ 16,
36
+ 17,
37
+ 18,
38
+ 19,
39
+ 20,
40
+ 21,
41
+ 22,
42
+ 23,
43
+ 24,
44
+ 25,
45
+ 26,
46
+ 27,
47
+ 28,
48
+ 29,
49
+ 30,
50
+ 31,
51
+ 32,
52
+ 33,
53
+ 34,
54
+ 35,
55
+ 36,
56
+ 37,
57
+ 38,
58
+ 39,
59
+ 40,
60
+ 41,
61
+ 42,
62
+ 43,
63
+ 44,
64
+ 45,
65
+ 46,
66
+ 47,
67
+ 48,
68
+ 49,
69
+ 50,
70
+ 51,
71
+ 52,
72
+ 53,
73
+ 54,
74
+ 55,
75
+ 56,
76
+ 57,
77
+ 58,
78
+ 59,
79
+ 60,
80
+ 61,
81
+ 62,
82
+ 63,
83
+ 64,
84
+ 65,
85
+ 66,
86
+ 67,
87
+ 68,
88
+ 69,
89
+ 70,
90
+ 71,
91
+ 72,
92
+ 73,
93
+ 74,
94
+ 75,
95
+ 76,
96
+ 77,
97
+ 78,
98
+ 79,
99
+ 80,
100
+ 81,
101
+ 82,
102
+ 83,
103
+ 84,
104
+ 85,
105
+ 86,
106
+ 87,
107
+ 88,
108
+ 89,
109
+ 90,
110
+ 91,
111
+ 92,
112
+ 93,
113
+ 94,
114
+ 95,
115
+ 96,
116
+ 97,
117
+ 98,
118
+ 99,
119
+ 100,
120
+ 101,
121
+ 102,
122
+ 103,
123
+ 104,
124
+ 105,
125
+ 106,
126
+ 107,
127
+ 108,
128
+ 109,
129
+ 110,
130
+ 111,
131
+ 112,
132
+ 113,
133
+ 114,
134
+ 115,
135
+ 116,
136
+ 117,
137
+ 118,
138
+ 119,
139
+ 120,
140
+ 121,
141
+ 122,
142
+ 123,
143
+ 124,
144
+ 125,
145
+ 126,
146
+ 127,
147
+ 128,
148
+ 129,
149
+ 130,
150
+ 131,
151
+ 132,
152
+ 133,
153
+ 134,
154
+ 135,
155
+ 136,
156
+ 137,
157
+ 138,
158
+ 139,
159
+ 140,
160
+ 141,
161
+ 142,
162
+ 143,
163
+ 144,
164
+ 145,
165
+ 146,
166
+ 147,
167
+ 148,
168
+ 149,
169
+ 150,
170
+ 151,
171
+ 152,
172
+ 153,
173
+ 154,
174
+ 155,
175
+ 156,
176
+ 157,
177
+ 158,
178
+ 159,
179
+ 160,
180
+ 161,
181
+ 162,
182
+ 163,
183
+ 164,
184
+ 165,
185
+ 166,
186
+ 167,
187
+ 168,
188
+ 169,
189
+ 170,
190
+ 171,
191
+ 172,
192
+ 173,
193
+ 174,
194
+ 175,
195
+ 176,
196
+ 177,
197
+ 178,
198
+ 179,
199
+ 180,
200
+ 181,
201
+ 182,
202
+ 183,
203
+ 184,
204
+ 185,
205
+ 186,
206
+ 187,
207
+ 188,
208
+ 189,
209
+ 190,
210
+ 191,
211
+ 192,
212
+ 193,
213
+ 194,
214
+ 195,
215
+ 196,
216
+ 197,
217
+ 198,
218
+ 199,
219
+ 200,
220
+ 201,
221
+ 202,
222
+ 203,
223
+ 204,
224
+ 205,
225
+ 206,
226
+ 207,
227
+ 208,
228
+ 209,
229
+ 210,
230
+ 211,
231
+ 212,
232
+ 213,
233
+ 214,
234
+ 215,
235
+ 216,
236
+ 217,
237
+ 218,
238
+ 219,
239
+ 220,
240
+ 221,
241
+ 222,
242
+ 223,
243
+ 224,
244
+ 225,
245
+ 226,
246
+ 227,
247
+ 228,
248
+ 229,
249
+ 230,
250
+ 231,
251
+ 232,
252
+ 233,
253
+ 234,
254
+ 235,
255
+ 236,
256
+ 237,
257
+ 238,
258
+ 239,
259
+ 240,
260
+ 241,
261
+ 242,
262
+ 243,
263
+ 244,
264
+ 245,
265
+ 246,
266
+ 247,
267
+ 248,
268
+ 249,
269
+ 250,
270
+ 251,
271
+ 252,
272
+ 253,
273
+ 254,
274
+ 255,
275
+ 256,
276
+ 257,
277
+ 258,
278
+ 259,
279
+ 260,
280
+ 261,
281
+ 262,
282
+ 263,
283
+ 264,
284
+ 265,
285
+ 266
286
+ ]
287
+ },
288
+ {
289
+ "weight_decay": 0.0,
290
+ "lr": 0.0,
291
+ "betas": [
292
+ 0.9,
293
+ 0.999
294
+ ],
295
+ "eps": 1e-08,
296
+ "amsgrad": false,
297
+ "maximize": false,
298
+ "foreach": null,
299
+ "capturable": false,
300
+ "differentiable": false,
301
+ "fused": null,
302
+ "decoupled_weight_decay": true,
303
+ "initial_lr": 0.0001,
304
+ "params": [
305
+ 267,
306
+ 268,
307
+ 269,
308
+ 270,
309
+ 271,
310
+ 272,
311
+ 273,
312
+ 274,
313
+ 275,
314
+ 276,
315
+ 277,
316
+ 278,
317
+ 279,
318
+ 280,
319
+ 281,
320
+ 282,
321
+ 283,
322
+ 284,
323
+ 285,
324
+ 286,
325
+ 287,
326
+ 288,
327
+ 289,
328
+ 290,
329
+ 291,
330
+ 292,
331
+ 293,
332
+ 294,
333
+ 295,
334
+ 296,
335
+ 297,
336
+ 298,
337
+ 299,
338
+ 300,
339
+ 301,
340
+ 302,
341
+ 303,
342
+ 304,
343
+ 305,
344
+ 306,
345
+ 307,
346
+ 308,
347
+ 309,
348
+ 310,
349
+ 311,
350
+ 312,
351
+ 313,
352
+ 314,
353
+ 315,
354
+ 316,
355
+ 317,
356
+ 318,
357
+ 319,
358
+ 320,
359
+ 321,
360
+ 322,
361
+ 323,
362
+ 324,
363
+ 325,
364
+ 326,
365
+ 327,
366
+ 328,
367
+ 329,
368
+ 330,
369
+ 331,
370
+ 332,
371
+ 333,
372
+ 334,
373
+ 335,
374
+ 336,
375
+ 337,
376
+ 338,
377
+ 339,
378
+ 340,
379
+ 341,
380
+ 342,
381
+ 343,
382
+ 344,
383
+ 345,
384
+ 346,
385
+ 347,
386
+ 348,
387
+ 349,
388
+ 350,
389
+ 351,
390
+ 352,
391
+ 353,
392
+ 354,
393
+ 355,
394
+ 356,
395
+ 357,
396
+ 358,
397
+ 359,
398
+ 360,
399
+ 361,
400
+ 362,
401
+ 363,
402
+ 364,
403
+ 365,
404
+ 366,
405
+ 367,
406
+ 368,
407
+ 369,
408
+ 370,
409
+ 371,
410
+ 372,
411
+ 373,
412
+ 374,
413
+ 375,
414
+ 376,
415
+ 377,
416
+ 378,
417
+ 379,
418
+ 380,
419
+ 381,
420
+ 382,
421
+ 383,
422
+ 384,
423
+ 385,
424
+ 386,
425
+ 387,
426
+ 388,
427
+ 389,
428
+ 390,
429
+ 391,
430
+ 392,
431
+ 393,
432
+ 394,
433
+ 395,
434
+ 396,
435
+ 397,
436
+ 398,
437
+ 399,
438
+ 400,
439
+ 401,
440
+ 402,
441
+ 403,
442
+ 404,
443
+ 405,
444
+ 406,
445
+ 407,
446
+ 408,
447
+ 409,
448
+ 410,
449
+ 411,
450
+ 412,
451
+ 413,
452
+ 414,
453
+ 415,
454
+ 416,
455
+ 417,
456
+ 418,
457
+ 419,
458
+ 420,
459
+ 421,
460
+ 422,
461
+ 423,
462
+ 424,
463
+ 425,
464
+ 426,
465
+ 427,
466
+ 428,
467
+ 429,
468
+ 430,
469
+ 431,
470
+ 432,
471
+ 433,
472
+ 434,
473
+ 435,
474
+ 436,
475
+ 437,
476
+ 438,
477
+ 439,
478
+ 440,
479
+ 441,
480
+ 442,
481
+ 443,
482
+ 444,
483
+ 445,
484
+ 446,
485
+ 447,
486
+ 448,
487
+ 449,
488
+ 450,
489
+ 451,
490
+ 452,
491
+ 453,
492
+ 454,
493
+ 455,
494
+ 456,
495
+ 457,
496
+ 458,
497
+ 459,
498
+ 460,
499
+ 461,
500
+ 462,
501
+ 463,
502
+ 464,
503
+ 465,
504
+ 466,
505
+ 467,
506
+ 468,
507
+ 469,
508
+ 470,
509
+ 471,
510
+ 472,
511
+ 473,
512
+ 474,
513
+ 475,
514
+ 476,
515
+ 477,
516
+ 478,
517
+ 479,
518
+ 480,
519
+ 481,
520
+ 482,
521
+ 483,
522
+ 484,
523
+ 485,
524
+ 486,
525
+ 487,
526
+ 488,
527
+ 489,
528
+ 490,
529
+ 491,
530
+ 492,
531
+ 493,
532
+ 494,
533
+ 495,
534
+ 496,
535
+ 497,
536
+ 498,
537
+ 499,
538
+ 500,
539
+ 501,
540
+ 502,
541
+ 503,
542
+ 504,
543
+ 505,
544
+ 506,
545
+ 507,
546
+ 508,
547
+ 509,
548
+ 510,
549
+ 511,
550
+ 512,
551
+ 513,
552
+ 514,
553
+ 515,
554
+ 516,
555
+ 517,
556
+ 518,
557
+ 519,
558
+ 520,
559
+ 521,
560
+ 522,
561
+ 523,
562
+ 524,
563
+ 525,
564
+ 526,
565
+ 527,
566
+ 528,
567
+ 529,
568
+ 530,
569
+ 531,
570
+ 532,
571
+ 533,
572
+ 534,
573
+ 535,
574
+ 536
575
+ ]
576
+ }
577
+ ]
Abs_6D/Baseline/checkpoints/060000/training_state/scheduler_state.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_lrs": [
3
+ 0.0001,
4
+ 0.0001
5
+ ],
6
+ "last_epoch": 60000,
7
+ "_step_count": 60001,
8
+ "_is_initial": false,
9
+ "_get_lr_called_within_step": false,
10
+ "_last_lr": [
11
+ 0.0,
12
+ 0.0
13
+ ],
14
+ "lr_lambdas": [
15
+ null,
16
+ null
17
+ ]
18
+ }
Abs_6D/Baseline/checkpoints/060000/training_state/training_step.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "step": 60000,
3
+ "num_processes": 1,
4
+ "batch_size": 64
5
+ }
Abs_6D/Baseline/wandb/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
Abs_6D/Baseline/wandb/debug.log ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
2
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_setup.py:_flush():81] Configure stats pid to 1450496
3
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug.log
5
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug-internal.log
6
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_init.py:init():766] calling init triggers
7
+ 2026-07-15 14:32:50,280 INFO MainThread:1450496 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
8
+ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/Abs_6D/Baseline', 'job_name': 'INSIGHT_6D_baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '1jmjb68j', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000', '_wandb': {}}
9
+ 2026-07-15 14:32:50,280 INFO MainThread:1450496 [wandb_init.py:init():814] starting backend
10
+ 2026-07-15 14:32:50,495 INFO MainThread:1450496 [wandb_init.py:init():829] sending inform_init request
11
+ 2026-07-15 14:32:50,989 INFO MainThread:1450496 [wandb_init.py:init():834] backend started and connected
12
+ 2026-07-15 14:32:50,999 INFO MainThread:1450496 [wandb_init.py:init():904] updated telemetry
13
+ 2026-07-15 14:32:51,006 INFO MainThread:1450496 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
14
+ 2026-07-15 14:32:51,687 INFO MainThread:1450496 [wandb_init.py:init():967] run resumed
15
+ 2026-07-15 14:32:51,692 INFO MainThread:1450496 [wandb_init.py:init():972] starting run threads in backend
16
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_console_start():2523] atexit reg
17
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_redirect():2373] redirect: wrap_raw
18
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_redirect():2442] Wrapping output streams.
19
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_redirect():2465] Redirects installed.
20
+ 2026-07-15 14:32:51,785 INFO MainThread:1450496 [wandb_init.py:init():1010] run started, returning control to user process
21
+ 2026-07-15 19:10:44,422 INFO wandb-AsyncioManager-main:1450496 [service_client.py:_forward_responses():122] Reached EOF.
22
+ 2026-07-15 19:10:44,422 INFO wandb-AsyncioManager-main:1450496 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
23
+ 2026-07-15 19:10:44,423 ERROR wandb-AsyncioManager-main:1450496 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
24
+ Traceback (most recent call last):
25
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
26
+ await fn()
27
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
28
+ await asyncio_compat.race(
29
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
30
+ async with open_task_group(race=True) as tg:
31
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
32
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
33
+ await anext(self.gen)
34
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
35
+ await task_group._wait_all(race=race, timeout=exit_timeout)
36
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
37
+ raise exc
38
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
39
+ result = await handle.wait_async(timeout=None)
40
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
41
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
42
+ response = await self._handle.wait_async(timeout=timeout)
43
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
44
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
45
+ raise HandleAbandonedError()
46
+ wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/files/output.log ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO 2026-07-14 16:12:42 db_utils.py:121 Logs will be synced with wandb.
2
+ INFO 2026-07-14 16:12:42 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/1jmjb68j
3
+ INFO 2026-07-14 16:12:42 ot_train.py:277 Creating dataset
4
+ INFO 2026-07-14 16:12:45 ot_train.py:311 Creating policy
5
+ Fetching 27 files: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 27/27 [00:00<00:00, 4171.90it/s]
6
+ `torch_dtype` is deprecated! Use `dtype` instead!
7
+ Loading weights: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1031/1031 [00:00<00:00, 2241.16it/s]
8
+ INFO 2026-07-14 16:12:51 ot_train.py:384 Creating optimizer and scheduler
9
+ INFO 2026-07-14 16:12:51 ot_train.py:416 Output dir: /home/ext_minje/groot_insight/Abs_6D/Baseline
10
+ INFO 2026-07-14 16:12:51 ot_train.py:423 cfg.steps=60000 (60K)
11
+ INFO 2026-07-14 16:12:51 ot_train.py:424 dataset.num_frames=218367 (218K)
12
+ INFO 2026-07-14 16:12:51 ot_train.py:425 dataset.num_episodes=4930
13
+ INFO 2026-07-14 16:12:51 ot_train.py:428 Effective batch size: 64 x 1 = 64
14
+ INFO 2026-07-14 16:12:51 ot_train.py:429 num_learnable_params=1620515968 (2B)
15
+ INFO 2026-07-14 16:12:51 ot_train.py:430 num_total_params=3144016000 (3B)
16
+ Training: 0%| | 0/60000 [00:00<?, ?step/s]INFO 2026-07-14 16:12:51 ot_train.py:581 Start offline training on a fixed dataset, with effective batch size: 64
17
+ Training: 0%| | 200/60000 [04:57<25:13:11, 1.52s/step]INFO 2026-07-14 16:17:48 ot_train.py:625 step:200 smpl:13K ep:289 epch:0.06 loss:0.924 grdn:1.272 lr:2.0e-05 updt_s:0.764 data_s:0.721 smp/s:43 mem_gb:39.61
18
+ Training: 1%| | 400/60000 [09:47<21:50:42, 1.32s/step]INFO 2026-07-14 16:22:38 ot_train.py:625 step:400 smpl:26K ep:578 epch:0.12 loss:0.258 grdn:1.552 lr:6.0e-05 updt_s:0.734 data_s:0.714 smp/s:44 mem_gb:39.63
19
+ Training: 1%| | 600/60000 [14:37<22:35:00, 1.37s/step]INFO 2026-07-14 16:27:28 ot_train.py:625 step:600 smpl:38K ep:867 epch:0.18 loss:0.199 grdn:1.034 lr:9.5e-05 updt_s:0.734 data_s:0.711 smp/s:44 mem_gb:39.63
20
+ Training: 1%|▏ | 800/60000 [19:30<23:53:10, 1.45s/step]INFO 2026-07-14 16:32:21 ot_train.py:625 step:800 smpl:51K ep:1K epch:0.23 loss:0.163 grdn:0.780 lr:1.0e-04 updt_s:0.731 data_s:0.731 smp/s:44 mem_gb:39.63
21
+ Training: 2%|▏ | 1000/60000 [24:22<26:08:09, 1.59s/step]INFO 2026-07-14 16:37:13 ot_train.py:625 step:1K smpl:64K ep:1K epch:0.29 loss:0.147 grdn:0.695 lr:1.0e-04 updt_s:0.735 data_s:0.721 smp/s:44 mem_gb:39.63
22
+ Training: 2%|▏ | 1200/60000 [29:14<23:30:02, 1.44s/step]INFO 2026-07-14 16:42:06 ot_train.py:625 step:1K smpl:77K ep:2K epch:0.35 loss:0.132 grdn:0.624 lr:1.0e-04 updt_s:0.741 data_s:0.719 smp/s:44 mem_gb:39.63
23
+ Training: 2%|▏ | 1400/60000 [34:02<22:35:14, 1.39s/step]INFO 2026-07-14 16:46:53 ot_train.py:625 step:1K smpl:90K ep:2K epch:0.41 loss:0.123 grdn:0.569 lr:1.0e-04 updt_s:0.723 data_s:0.712 smp/s:45 mem_gb:39.63
24
+ Training: 3%|β–Ž | 1600/60000 [38:48<24:19:48, 1.50s/step]INFO 2026-07-14 16:51:40 ot_train.py:625 step:2K smpl:102K ep:2K epch:0.47 loss:0.119 grdn:0.548 lr:1.0e-04 updt_s:0.731 data_s:0.698 smp/s:45 mem_gb:39.63
25
+ Training: 3%|β–Ž | 1800/60000 [43:34<21:35:33, 1.34s/step]INFO 2026-07-14 16:56:26 ot_train.py:625 step:2K smpl:115K ep:3K epch:0.53 loss:0.115 grdn:0.516 lr:1.0e-04 updt_s:0.728 data_s:0.700 smp/s:45 mem_gb:39.63
26
+ Training: 3%|β–Ž | 2000/60000 [48:21<22:37:52, 1.40s/step]INFO 2026-07-14 17:01:12 ot_train.py:625 step:2K smpl:128K ep:3K epch:0.59 loss:0.110 grdn:0.486 lr:1.0e-04 updt_s:0.734 data_s:0.697 smp/s:45 mem_gb:39.63
27
+ Training: 4%|β–Ž | 2200/60000 [53:16<27:27:37, 1.71s/step]INFO 2026-07-14 17:06:07 ot_train.py:625 step:2K smpl:141K ep:3K epch:0.64 loss:0.108 grdn:0.488 lr:1.0e-04 updt_s:0.736 data_s:0.735 smp/s:43 mem_gb:39.63
28
+ Training: 4%|▍ | 2400/60000 [57:52<20:49:34, 1.30s/step]INFO 2026-07-14 17:10:43 ot_train.py:625 step:2K smpl:154K ep:3K epch:0.70 loss:0.104 grdn:0.448 lr:1.0e-04 updt_s:0.722 data_s:0.654 smp/s:47 mem_gb:39.62
29
+ Training: 4%|▍ | 2600/60000 [1:02:13<20:53:12, 1.31s/step]INFO 2026-07-14 17:15:04 ot_train.py:625 step:3K smpl:166K ep:4K epch:0.76 loss:0.105 grdn:0.451 lr:1.0e-04 updt_s:0.684 data_s:0.617 smp/s:49 mem_gb:39.64
30
+ Training: 5%|▍ | 2800/60000 [1:07:04<23:58:13, 1.51s/step]INFO 2026-07-14 17:19:55 ot_train.py:625 step:3K smpl:179K ep:4K epch:0.82 loss:0.100 grdn:0.426 lr:1.0e-04 updt_s:0.736 data_s:0.715 smp/s:44 mem_gb:39.64
31
+ Training: 5%|β–Œ | 3000/60000 [1:12:20<24:00:49, 1.52s/step]INFO 2026-07-14 17:25:11 ot_train.py:625 step:3K smpl:192K ep:4K epch:0.88 loss:0.099 grdn:0.413 lr:1.0e-04 updt_s:0.772 data_s:0.803 smp/s:41 mem_gb:39.64
32
+ Training: 5%|β–Œ | 3200/60000 [1:16:55<22:01:57, 1.40s/step]INFO 2026-07-14 17:29:46 ot_train.py:625 step:3K smpl:205K ep:5K epch:0.94 loss:0.098 grdn:0.405 lr:1.0e-04 updt_s:0.708 data_s:0.667 smp/s:47 mem_gb:39.64
33
+ Training: 6%|β–Œ | 3400/60000 [1:22:12<25:28:17, 1.62s/step]INFO 2026-07-14 17:35:03 ot_train.py:625 step:3K smpl:218K ep:5K epch:1.00 loss:0.095 grdn:0.389 lr:9.9e-05 updt_s:0.767 data_s:0.811 smp/s:41 mem_gb:39.64
34
+ Training: 6%|β–Œ | 3600/60000 [1:27:35<22:56:31, 1.46s/step]INFO 2026-07-14 17:40:26 ot_train.py:625 step:4K smpl:230K ep:5K epch:1.06 loss:0.094 grdn:0.384 lr:9.9e-05 updt_s:0.778 data_s:0.835 smp/s:40 mem_gb:39.64
35
+ Training: 6%|β–‹ | 3800/60000 [1:32:56<25:08:26, 1.61s/step]INFO 2026-07-14 17:45:47 ot_train.py:625 step:4K smpl:243K ep:5K epch:1.11 loss:0.094 grdn:0.389 lr:9.9e-05 updt_s:0.780 data_s:0.822 smp/s:40 mem_gb:39.64
36
+ Training: 7%|β–‹ | 4000/60000 [1:38:22<26:54:50, 1.73s/step]INFO 2026-07-14 17:51:13 ot_train.py:625 step:4K smpl:256K ep:6K epch:1.17 loss:0.093 grdn:0.374 lr:9.9e-05 updt_s:0.792 data_s:0.836 smp/s:39 mem_gb:39.64
37
+ Training: 7%|β–‹ | 4200/60000 [1:43:51<25:40:34, 1.66s/step]INFO 2026-07-14 17:56:42 ot_train.py:625 step:4K smpl:269K ep:6K epch:1.23 loss:0.090 grdn:0.355 lr:9.9e-05 updt_s:0.785 data_s:0.854 smp/s:39 mem_gb:39.64
38
+ Training: 7%|β–‹ | 4400/60000 [1:49:18<25:19:11, 1.64s/step]INFO 2026-07-14 18:02:09 ot_train.py:625 step:4K smpl:282K ep:6K epch:1.29 loss:0.090 grdn:0.361 lr:9.9e-05 updt_s:0.788 data_s:0.843 smp/s:39 mem_gb:39.63
39
+ Training: 8%|β–Š | 4600/60000 [1:54:41<25:34:40, 1.66s/step]INFO 2026-07-14 18:07:32 ot_train.py:625 step:5K smpl:294K ep:7K epch:1.35 loss:0.090 grdn:0.359 lr:9.9e-05 updt_s:0.777 data_s:0.834 smp/s:40 mem_gb:39.62
40
+ Training: 8%|β–Š | 4800/60000 [2:00:00<24:06:01, 1.57s/step]INFO 2026-07-14 18:12:51 ot_train.py:625 step:5K smpl:307K ep:7K epch:1.41 loss:0.089 grdn:0.369 lr:9.9e-05 updt_s:0.769 data_s:0.823 smp/s:40 mem_gb:39.64
41
+ Training: 8%|β–Š | 5000/60000 [2:04:54<18:21:51, 1.20s/step]INFO 2026-07-14 18:17:46 ot_train.py:625 step:5K smpl:320K ep:7K epch:1.47 loss:0.089 grdn:0.357 lr:9.9e-05 updt_s:0.745 data_s:0.725 smp/s:44 mem_gb:39.64
42
+ Training: 9%|β–Š | 5200/60000 [2:09:11<22:04:04, 1.45s/step]INFO 2026-07-14 18:22:02 ot_train.py:625 step:5K smpl:333K ep:8K epch:1.52 loss:0.086 grdn:0.338 lr:9.9e-05 updt_s:0.688 data_s:0.589 smp/s:50 mem_gb:39.64
43
+ Training: 9%|β–‰ | 5400/60000 [2:13:52<18:51:33, 1.24s/step]INFO 2026-07-14 18:26:43 ot_train.py:625 step:5K smpl:346K ep:8K epch:1.58 loss:0.084 grdn:0.322 lr:9.8e-05 updt_s:0.728 data_s:0.675 smp/s:46 mem_gb:39.64
44
+ Training: 9%|β–‰ | 5600/60000 [2:18:31<21:32:04, 1.43s/step]INFO 2026-07-14 18:31:22 ot_train.py:625 step:6K smpl:358K ep:8K epch:1.64 loss:0.085 grdn:0.331 lr:9.8e-05 updt_s:0.719 data_s:0.674 smp/s:46 mem_gb:39.64
45
+ Training: 10%|β–‰ | 5800/60000 [2:23:00<19:34:55, 1.30s/step]INFO 2026-07-14 18:35:52 ot_train.py:625 step:6K smpl:371K ep:8K epch:1.70 loss:0.085 grdn:0.334 lr:9.8e-05 updt_s:0.702 data_s:0.642 smp/s:48 mem_gb:39.64
46
+ Training: 10%|β–ˆ | 6000/60000 [2:27:14<18:52:42, 1.26s/step]INFO 2026-07-14 18:40:05 ot_train.py:625 step:6K smpl:384K ep:9K epch:1.76 loss:0.082 grdn:0.329 lr:9.8e-05 updt_s:0.669 data_s:0.596 smp/s:51 mem_gb:39.63
47
+ Training: 10%|β–ˆ | 6200/60000 [2:31:29<18:57:12, 1.27s/step]INFO 2026-07-14 18:44:20 ot_train.py:625 step:6K smpl:397K ep:9K epch:1.82 loss:0.085 grdn:0.346 lr:9.8e-05 updt_s:0.673 data_s:0.600 smp/s:50 mem_gb:39.64
48
+ Training: 11%|β–ˆ | 6400/60000 [2:35:43<19:12:50, 1.29s/step]INFO 2026-07-14 18:48:34 ot_train.py:625 step:6K smpl:410K ep:9K epch:1.88 loss:0.084 grdn:0.332 lr:9.8e-05 updt_s:0.670 data_s:0.596 smp/s:51 mem_gb:39.64
49
+ Training: 11%|β–ˆ | 6600/60000 [2:39:53<18:06:25, 1.22s/step]INFO 2026-07-14 18:52:45 ot_train.py:625 step:7K smpl:422K ep:10K epch:1.93 loss:0.079 grdn:0.300 lr:9.8e-05 updt_s:0.669 data_s:0.582 smp/s:51 mem_gb:39.64
50
+ Training: 11%|β–ˆβ– | 6800/60000 [2:44:06<18:20:51, 1.24s/step]INFO 2026-07-14 18:56:57 ot_train.py:625 step:7K smpl:435K ep:10K epch:1.99 loss:0.080 grdn:0.314 lr:9.7e-05 updt_s:0.665 data_s:0.596 smp/s:51 mem_gb:39.62
51
+ Training: 12%|β–ˆβ– | 7000/60000 [2:48:25<18:42:17, 1.27s/step]INFO 2026-07-14 19:01:16 ot_train.py:625 step:7K smpl:448K ep:10K epch:2.05 loss:0.078 grdn:0.316 lr:9.7e-05 updt_s:0.682 data_s:0.609 smp/s:50 mem_gb:39.64
52
+ Training: 12%|β–ˆβ– | 7200/60000 [2:52:42<19:19:08, 1.32s/step]INFO 2026-07-14 19:05:33 ot_train.py:625 step:7K smpl:461K ep:10K epch:2.11 loss:0.078 grdn:0.307 lr:9.7e-05 updt_s:0.686 data_s:0.593 smp/s:50 mem_gb:39.64
53
+ Training: 12%|β–ˆβ– | 7400/60000 [2:57:04<18:49:23, 1.29s/step]INFO 2026-07-14 19:09:55 ot_train.py:625 step:7K smpl:474K ep:11K epch:2.17 loss:0.077 grdn:0.291 lr:9.7e-05 updt_s:0.693 data_s:0.617 smp/s:49 mem_gb:39.64
54
+ Training: 13%|β–ˆβ–Ž | 7600/60000 [3:01:28<18:51:21, 1.30s/step]INFO 2026-07-14 19:14:19 ot_train.py:625 step:8K smpl:486K ep:11K epch:2.23 loss:0.079 grdn:0.329 lr:9.7e-05 updt_s:0.694 data_s:0.621 smp/s:49 mem_gb:39.64
55
+ Training: 13%|β–ˆβ–Ž | 7800/60000 [3:05:46<18:19:01, 1.26s/step]INFO 2026-07-14 19:18:38 ot_train.py:625 step:8K smpl:499K ep:11K epch:2.29 loss:0.080 grdn:0.329 lr:9.6e-05 updt_s:0.683 data_s:0.606 smp/s:50 mem_gb:39.64
56
+ Training: 13%|β–ˆβ–Ž | 8000/60000 [3:10:03<18:01:27, 1.25s/step]INFO 2026-07-14 19:22:54 ot_train.py:625 step:8K smpl:512K ep:12K epch:2.34 loss:0.081 grdn:0.327 lr:9.6e-05 updt_s:0.686 data_s:0.592 smp/s:50 mem_gb:39.64
57
+ Training: 14%|β–ˆβ–Ž | 8200/60000 [3:14:22<18:28:54, 1.28s/step]INFO 2026-07-14 19:27:13 ot_train.py:625 step:8K smpl:525K ep:12K epch:2.40 loss:0.077 grdn:0.301 lr:9.6e-05 updt_s:0.689 data_s:0.604 smp/s:49 mem_gb:39.64
58
+ Training: 14%|β–ˆβ– | 8400/60000 [3:18:42<19:26:41, 1.36s/step]INFO 2026-07-14 19:31:34 ot_train.py:625 step:8K smpl:538K ep:12K epch:2.46 loss:0.075 grdn:0.294 lr:9.6e-05 updt_s:0.689 data_s:0.612 smp/s:49 mem_gb:39.64
59
+ Training: 14%|β–ˆβ– | 8600/60000 [3:23:03<18:46:25, 1.31s/step]INFO 2026-07-14 19:35:54 ot_train.py:625 step:9K smpl:550K ep:12K epch:2.52 loss:0.076 grdn:0.297 lr:9.6e-05 updt_s:0.687 data_s:0.613 smp/s:49 mem_gb:39.64
60
+ Training: 15%|β–ˆβ– | 8800/60000 [3:27:30<18:50:26, 1.32s/step]INFO 2026-07-14 19:40:21 ot_train.py:625 step:9K smpl:563K ep:13K epch:2.58 loss:0.075 grdn:0.305 lr:9.5e-05 updt_s:0.711 data_s:0.622 smp/s:48 mem_gb:39.63
61
+ Training: 15%|β–ˆβ–Œ | 9000/60000 [3:31:54<18:56:39, 1.34s/step]INFO 2026-07-14 19:44:45 ot_train.py:625 step:9K smpl:576K ep:13K epch:2.64 loss:0.075 grdn:0.296 lr:9.5e-05 updt_s:0.690 data_s:0.626 smp/s:49 mem_gb:39.64
62
+ Training: 15%|β–ˆβ–Œ | 9200/60000 [3:36:15<18:15:36, 1.29s/step]INFO 2026-07-14 19:49:06 ot_train.py:625 step:9K smpl:589K ep:13K epch:2.70 loss:0.072 grdn:0.288 lr:9.5e-05 updt_s:0.686 data_s:0.615 smp/s:49 mem_gb:39.62
63
+ Training: 16%|β–ˆβ–Œ | 9400/60000 [3:40:27<17:40:11, 1.26s/step]INFO 2026-07-14 19:53:18 ot_train.py:625 step:9K smpl:602K ep:14K epch:2.75 loss:0.076 grdn:0.331 lr:9.5e-05 updt_s:0.670 data_s:0.587 smp/s:51 mem_gb:39.64
64
+ Training: 16%|β–ˆβ–Œ | 9600/60000 [3:44:45<17:53:30, 1.28s/step]INFO 2026-07-14 19:57:36 ot_train.py:625 step:10K smpl:614K ep:14K epch:2.81 loss:0.073 grdn:0.299 lr:9.4e-05 updt_s:0.682 data_s:0.605 smp/s:50 mem_gb:39.64
65
+ Training: 16%|β–ˆβ–‹ | 9800/60000 [3:49:05<17:46:19, 1.27s/step]INFO 2026-07-14 20:01:56 ot_train.py:625 step:10K smpl:627K ep:14K epch:2.87 loss:0.075 grdn:0.316 lr:9.4e-05 updt_s:0.682 data_s:0.615 smp/s:49 mem_gb:39.64
66
+ Training: 17%|β–ˆβ–‹ | 10000/60000 [3:53:23<17:37:11, 1.27s/step]INFO 2026-07-14 20:06:14 ot_train.py:625 step:10K smpl:640K ep:14K epch:2.93 loss:0.071 grdn:0.292 lr:9.4e-05 updt_s:0.681 data_s:0.609 smp/s:50 mem_gb:39.64
67
+ Training: 17%|β–ˆβ–‹ | 10200/60000 [3:57:43<18:16:00, 1.32s/step]INFO 2026-07-14 20:10:34 ot_train.py:625 step:10K smpl:653K ep:15K epch:2.99 loss:0.075 grdn:0.321 lr:9.4e-05 updt_s:0.681 data_s:0.616 smp/s:49 mem_gb:39.63
68
+ Training: 17%|β–ˆβ–‹ | 10400/60000 [4:02:02<17:42:10, 1.28s/step]INFO 2026-07-14 20:14:53 ot_train.py:625 step:10K smpl:666K ep:15K epch:3.05 loss:0.072 grdn:0.289 lr:9.3e-05 updt_s:0.679 data_s:0.611 smp/s:50 mem_gb:39.64
69
+ Training: 18%|β–ˆβ–Š | 10600/60000 [4:06:08<16:46:22, 1.22s/step]INFO 2026-07-14 20:19:00 ot_train.py:625 step:11K smpl:678K ep:15K epch:3.11 loss:0.071 grdn:0.282 lr:9.3e-05 updt_s:0.660 data_s:0.571 smp/s:52 mem_gb:39.64
70
+ Training: 18%|β–ˆβ–Š | 10800/60000 [4:10:16<16:44:40, 1.23s/step]INFO 2026-07-14 20:23:07 ot_train.py:625 step:11K smpl:691K ep:16K epch:3.17 loss:0.071 grdn:0.279 lr:9.3e-05 updt_s:0.660 data_s:0.574 smp/s:52 mem_gb:39.63
71
+ Training: 18%|β–ˆβ–Š | 11000/60000 [4:14:24<16:49:58, 1.24s/step]INFO 2026-07-14 20:27:16 ot_train.py:625 step:11K smpl:704K ep:16K epch:3.22 loss:0.067 grdn:0.270 lr:9.3e-05 updt_s:0.663 data_s:0.578 smp/s:52 mem_gb:39.64
72
+ Training: 19%|β–ˆβ–Š | 11200/60000 [4:18:36<16:38:47, 1.23s/step]INFO 2026-07-14 20:31:27 ot_train.py:625 step:11K smpl:717K ep:16K epch:3.28 loss:0.067 grdn:0.274 lr:9.2e-05 updt_s:0.665 data_s:0.592 smp/s:51 mem_gb:39.64
73
+ Training: 19%|β–ˆβ–‰ | 11400/60000 [4:22:47<16:26:56, 1.22s/step]INFO 2026-07-14 20:35:38 ot_train.py:625 step:11K smpl:730K ep:16K epch:3.34 loss:0.070 grdn:0.292 lr:9.2e-05 updt_s:0.658 data_s:0.591 smp/s:51 mem_gb:39.62
74
+ Training: 19%|β–ˆβ–‰ | 11600/60000 [4:27:00<17:21:50, 1.29s/step]INFO 2026-07-14 20:39:51 ot_train.py:625 step:12K smpl:742K ep:17K epch:3.40 loss:0.067 grdn:0.283 lr:9.2e-05 updt_s:0.667 data_s:0.595 smp/s:51 mem_gb:39.64
75
+ Training: 20%|β–ˆβ–‰ | 11800/60000 [4:31:17<16:16:30, 1.22s/step]INFO 2026-07-14 20:44:08 ot_train.py:625 step:12K smpl:755K ep:17K epch:3.46 loss:0.070 grdn:0.290 lr:9.2e-05 updt_s:0.675 data_s:0.608 smp/s:50 mem_gb:39.64
76
+ Training: 20%|β–ˆβ–ˆ | 12000/60000 [4:35:26<16:27:26, 1.23s/step]INFO 2026-07-14 20:48:17 ot_train.py:625 step:12K smpl:768K ep:17K epch:3.52 loss:0.071 grdn:0.295 lr:9.1e-05 updt_s:0.657 data_s:0.587 smp/s:51 mem_gb:39.64
77
+ Training: 20%|β–ˆβ–ˆ | 12200/60000 [4:39:32<17:11:03, 1.29s/step]INFO 2026-07-14 20:52:23 ot_train.py:625 step:12K smpl:781K ep:18K epch:3.58 loss:0.067 grdn:0.274 lr:9.1e-05 updt_s:0.655 data_s:0.570 smp/s:52 mem_gb:39.64
78
+ Training: 21%|β–ˆοΏ½οΏ½ | 12400/60000 [4:43:39<16:03:48, 1.21s/step]INFO 2026-07-14 20:56:30 ot_train.py:625 step:12K smpl:794K ep:18K epch:3.63 loss:0.069 grdn:0.297 lr:9.1e-05 updt_s:0.657 data_s:0.576 smp/s:52 mem_gb:39.64
79
+ Training: 21%|β–ˆβ–ˆ | 12600/60000 [4:47:50<16:04:53, 1.22s/step]INFO 2026-07-14 21:00:41 ot_train.py:625 step:13K smpl:806K ep:18K epch:3.69 loss:0.066 grdn:0.270 lr:9.0e-05 updt_s:0.661 data_s:0.591 smp/s:51 mem_gb:39.64
80
+ Training: 21%|β–ˆβ–ˆβ– | 12800/60000 [4:52:03<16:09:50, 1.23s/step]INFO 2026-07-14 21:04:54 ot_train.py:625 step:13K smpl:819K ep:18K epch:3.75 loss:0.066 grdn:0.275 lr:9.0e-05 updt_s:0.666 data_s:0.596 smp/s:51 mem_gb:39.64
81
+ Training: 22%|β–ˆβ–ˆβ– | 13000/60000 [4:56:17<17:16:44, 1.32s/step]INFO 2026-07-14 21:09:08 ot_train.py:625 step:13K smpl:832K ep:19K epch:3.81 loss:0.067 grdn:0.284 lr:9.0e-05 updt_s:0.673 data_s:0.594 smp/s:50 mem_gb:39.63
82
+ Training: 22%|β–ˆβ–ˆβ– | 13200/60000 [5:00:26<16:13:10, 1.25s/step]INFO 2026-07-14 21:13:17 ot_train.py:625 step:13K smpl:845K ep:19K epch:3.87 loss:0.070 grdn:0.306 lr:8.9e-05 updt_s:0.661 data_s:0.583 smp/s:51 mem_gb:39.63
83
+ Training: 22%|β–ˆβ–ˆβ– | 13400/60000 [5:04:37<15:56:23, 1.23s/step]INFO 2026-07-14 21:17:29 ot_train.py:625 step:13K smpl:858K ep:19K epch:3.93 loss:0.068 grdn:0.301 lr:8.9e-05 updt_s:0.668 data_s:0.587 smp/s:51 mem_gb:39.64
84
+ Training: 23%|β–ˆβ–ˆβ–Ž | 13600/60000 [5:08:44<15:34:44, 1.21s/step]INFO 2026-07-14 21:21:35 ot_train.py:625 step:14K smpl:870K ep:20K epch:3.99 loss:0.067 grdn:0.281 lr:8.9e-05 updt_s:0.652 data_s:0.578 smp/s:52 mem_gb:39.62
85
+ Training: 23%|β–ˆβ–ˆβ–Ž | 13800/60000 [5:12:51<16:02:46, 1.25s/step]INFO 2026-07-14 21:25:42 ot_train.py:625 step:14K smpl:883K ep:20K epch:4.04 loss:0.065 grdn:0.268 lr:8.8e-05 updt_s:0.656 data_s:0.575 smp/s:52 mem_gb:39.64
86
+ Training: 23%|β–ˆβ–ˆβ–Ž | 14000/60000 [5:17:03<15:27:28, 1.21s/step]INFO 2026-07-14 21:29:54 ot_train.py:625 step:14K smpl:896K ep:20K epch:4.10 loss:0.064 grdn:0.285 lr:8.8e-05 updt_s:0.665 data_s:0.591 smp/s:51 mem_gb:39.64
87
+ Training: 24%|β–ˆβ–ˆβ–Ž | 14200/60000 [5:21:15<15:36:13, 1.23s/step]INFO 2026-07-14 21:34:06 ot_train.py:625 step:14K smpl:909K ep:21K epch:4.16 loss:0.066 grdn:0.281 lr:8.8e-05 updt_s:0.659 data_s:0.599 smp/s:51 mem_gb:39.64
88
+ Training: 24%|β–ˆβ–ˆβ– | 14400/60000 [5:25:33<16:34:18, 1.31s/step]INFO 2026-07-14 21:38:24 ot_train.py:625 step:14K smpl:922K ep:21K epch:4.22 loss:0.064 grdn:0.272 lr:8.7e-05 updt_s:0.682 data_s:0.605 smp/s:50 mem_gb:39.64
89
+ Training: 24%|β–ˆβ–ˆβ– | 14600/60000 [5:29:45<15:40:37, 1.24s/step]INFO 2026-07-14 21:42:36 ot_train.py:625 step:15K smpl:934K ep:21K epch:4.28 loss:0.067 grdn:0.315 lr:8.7e-05 updt_s:0.665 data_s:0.593 smp/s:51 mem_gb:39.63
90
+ Training: 25%|β–ˆβ–ˆβ– | 14800/60000 [5:34:00<15:40:37, 1.25s/step]INFO 2026-07-14 21:46:51 ot_train.py:625 step:15K smpl:947K ep:21K epch:4.34 loss:0.064 grdn:0.279 lr:8.7e-05 updt_s:0.671 data_s:0.599 smp/s:50 mem_gb:39.63
91
+ Training: 25%|β–ˆβ–ˆβ–Œ | 15000/60000 [5:38:06<15:14:31, 1.22s/step]INFO 2026-07-14 21:50:57 ot_train.py:625 step:15K smpl:960K ep:22K epch:4.40 loss:0.064 grdn:0.267 lr:8.6e-05 updt_s:0.655 data_s:0.572 smp/s:52 mem_gb:39.64
92
+ Training: 25%|β–ˆβ–ˆβ–Œ | 15200/60000 [5:42:12<14:50:06, 1.19s/step]INFO 2026-07-14 21:55:03 ot_train.py:625 step:15K smpl:973K ep:22K epch:4.45 loss:0.070 grdn:0.321 lr:8.6e-05 updt_s:0.657 data_s:0.571 smp/s:52 mem_gb:39.64
93
+ Training: 26%|β–ˆβ–ˆβ–Œ | 15400/60000 [5:46:20<15:10:47, 1.23s/step]INFO 2026-07-14 21:59:12 ot_train.py:625 step:15K smpl:986K ep:22K epch:4.51 loss:0.064 grdn:0.269 lr:8.5e-05 updt_s:0.660 data_s:0.581 smp/s:52 mem_gb:39.64
94
+ Training: 26%|β–ˆβ–ˆβ–Œ | 15600/60000 [5:50:34<15:03:47, 1.22s/step]INFO 2026-07-14 22:03:25 ot_train.py:625 step:16K smpl:998K ep:23K epch:4.57 loss:0.063 grdn:0.274 lr:8.5e-05 updt_s:0.662 data_s:0.604 smp/s:51 mem_gb:39.64
95
+ Training: 26%|β–ˆβ–ˆβ–‹ | 15800/60000 [5:54:50<19:01:44, 1.55s/step]INFO 2026-07-14 22:07:42 ot_train.py:625 step:16K smpl:1M ep:23K epch:4.63 loss:0.062 grdn:0.271 lr:8.5e-05 updt_s:0.671 data_s:0.607 smp/s:50 mem_gb:39.62
96
+ Training: 27%|β–ˆβ–ˆβ–‹ | 16000/60000 [5:59:04<14:59:09, 1.23s/step]INFO 2026-07-14 22:11:55 ot_train.py:625 step:16K smpl:1M ep:23K epch:4.69 loss:0.061 grdn:0.272 lr:8.4e-05 updt_s:0.674 data_s:0.592 smp/s:51 mem_gb:39.64
97
+ Training: 27%|β–ˆβ–ˆβ–‹ | 16200/60000 [6:03:18<15:56:38, 1.31s/step]INFO 2026-07-14 22:16:09 ot_train.py:625 step:16K smpl:1M ep:23K epch:4.75 loss:0.060 grdn:0.258 lr:8.4e-05 updt_s:0.668 data_s:0.598 smp/s:51 mem_gb:39.64
98
+ Training: 27%|β–ˆβ–ˆβ–‹ | 16400/60000 [6:07:27<15:43:33, 1.30s/step]INFO 2026-07-14 22:20:19 ot_train.py:625 step:16K smpl:1M ep:24K epch:4.81 loss:0.062 grdn:0.274 lr:8.4e-05 updt_s:0.660 data_s:0.585 smp/s:51 mem_gb:39.64
99
+ Training: 28%|β–ˆβ–ˆβ–Š | 16600/60000 [6:11:43<15:50:06, 1.31s/step]INFO 2026-07-14 22:24:35 ot_train.py:625 step:17K smpl:1M ep:24K epch:4.87 loss:0.060 grdn:0.270 lr:8.3e-05 updt_s:0.677 data_s:0.599 smp/s:50 mem_gb:39.63
100
+ Training: 28%|β–ˆβ–ˆβ–Š | 16800/60000 [6:16:05<15:21:36, 1.28s/step]INFO 2026-07-14 22:28:56 ot_train.py:625 step:17K smpl:1M ep:24K epch:4.92 loss:0.061 grdn:0.254 lr:8.3e-05 updt_s:0.685 data_s:0.618 smp/s:49 mem_gb:39.64
101
+ Training: 28%|β–ˆβ–ˆβ–Š | 17000/60000 [6:20:21<15:33:00, 1.30s/step]INFO 2026-07-14 22:33:12 ot_train.py:625 step:17K smpl:1M ep:25K epch:4.98 loss:0.061 grdn:0.262 lr:8.2e-05 updt_s:0.687 data_s:0.589 smp/s:50 mem_gb:39.64
102
+ Training: 29%|β–ˆβ–ˆβ–Š | 17200/60000 [6:24:31<14:26:42, 1.22s/step]INFO 2026-07-14 22:37:22 ot_train.py:625 step:17K smpl:1M ep:25K epch:5.04 loss:0.063 grdn:0.303 lr:8.2e-05 updt_s:0.676 data_s:0.573 smp/s:51 mem_gb:39.64
103
+ Training: 29%|β–ˆβ–ˆβ–‰ | 17400/60000 [6:28:40<14:54:34, 1.26s/step]INFO 2026-07-14 22:41:32 ot_train.py:625 step:17K smpl:1M ep:25K epch:5.10 loss:0.060 grdn:0.271 lr:8.2e-05 updt_s:0.673 data_s:0.571 smp/s:51 mem_gb:39.63
104
+ Training: 29%|β–ˆβ–ˆβ–‰ | 17600/60000 [6:32:50<15:00:30, 1.27s/step]INFO 2026-07-14 22:45:41 ot_train.py:625 step:18K smpl:1M ep:25K epch:5.16 loss:0.059 grdn:0.268 lr:8.1e-05 updt_s:0.676 data_s:0.568 smp/s:51 mem_gb:39.64
105
+ Training: 30%|β–ˆβ–ˆβ–‰ | 17800/60000 [6:37:01<14:37:09, 1.25s/step]INFO 2026-07-14 22:49:52 ot_train.py:625 step:18K smpl:1M ep:26K epch:5.22 loss:0.060 grdn:0.260 lr:8.1e-05 updt_s:0.670 data_s:0.581 smp/s:51 mem_gb:39.64
106
+ Training: 30%|β–ˆβ–ˆβ–ˆ | 18000/60000 [6:41:10<15:23:22, 1.32s/step]INFO 2026-07-14 22:54:01 ot_train.py:625 step:18K smpl:1M ep:26K epch:5.28 loss:0.060 grdn:0.281 lr:8.0e-05 updt_s:0.674 data_s:0.571 smp/s:51 mem_gb:39.64
107
+ Training: 30%|β–ˆβ–ˆβ–ˆ | 18200/60000 [6:45:20<14:02:08, 1.21s/step]INFO 2026-07-14 22:58:11 ot_train.py:625 step:18K smpl:1M ep:26K epch:5.33 loss:0.060 grdn:0.265 lr:8.0e-05 updt_s:0.669 data_s:0.577 smp/s:51 mem_gb:39.62
108
+ Training: 31%|β–ˆβ–ˆβ–ˆ | 18400/60000 [6:49:43<14:36:21, 1.26s/step]INFO 2026-07-14 23:02:34 ot_train.py:625 step:18K smpl:1M ep:27K epch:5.39 loss:0.057 grdn:0.259 lr:7.9e-05 updt_s:0.701 data_s:0.611 smp/s:49 mem_gb:39.64
109
+ Training: 31%|β–ˆβ–ˆβ–ˆ | 18600/60000 [6:54:06<14:11:09, 1.23s/step]INFO 2026-07-14 23:06:57 ot_train.py:625 step:19K smpl:1M ep:27K epch:5.45 loss:0.059 grdn:0.263 lr:7.9e-05 updt_s:0.699 data_s:0.612 smp/s:49 mem_gb:39.63
110
+ Training: 31%|β–ˆβ–ˆβ–ˆβ– | 18800/60000 [6:58:35<15:03:04, 1.32s/step]INFO 2026-07-14 23:11:27 ot_train.py:625 step:19K smpl:1M ep:27K epch:5.51 loss:0.058 grdn:0.273 lr:7.9e-05 updt_s:0.712 data_s:0.632 smp/s:48 mem_gb:39.64
111
+ Training: 32%|β–ˆβ–ˆβ–ˆβ– | 19000/60000 [7:03:01<14:58:10, 1.31s/step]INFO 2026-07-14 23:15:52 ot_train.py:625 step:19K smpl:1M ep:27K epch:5.57 loss:0.059 grdn:0.265 lr:7.8e-05 updt_s:0.694 data_s:0.631 smp/s:48 mem_gb:39.64
112
+ Training: 32%|β–ˆβ–ˆβ–ˆβ– | 19200/60000 [7:07:23<13:38:45, 1.20s/step]INFO 2026-07-14 23:20:15 ot_train.py:625 step:19K smpl:1M ep:28K epch:5.63 loss:0.059 grdn:0.262 lr:7.8e-05 updt_s:0.688 data_s:0.620 smp/s:49 mem_gb:39.64
113
+ Training: 32%|β–ˆβ–ˆβ–ˆβ– | 19400/60000 [7:11:17<12:53:09, 1.14s/step]INFO 2026-07-14 23:24:08 ot_train.py:625 step:19K smpl:1M ep:28K epch:5.69 loss:0.058 grdn:0.265 lr:7.7e-05 updt_s:0.626 data_s:0.540 smp/s:55 mem_gb:39.63
114
+ Training: 33%|β–ˆβ–ˆβ–ˆβ–Ž | 19600/60000 [7:15:11<14:19:26, 1.28s/step]INFO 2026-07-14 23:28:02 ot_train.py:625 step:20K smpl:1M ep:28K epch:5.74 loss:0.056 grdn:0.261 lr:7.7e-05 updt_s:0.630 data_s:0.535 smp/s:55 mem_gb:39.64
115
+ Training: 33%|β–ˆβ–ˆβ–ˆβ–Ž | 19800/60000 [7:19:32<16:27:23, 1.47s/step]INFO 2026-07-14 23:32:23 ot_train.py:625 step:20K smpl:1M ep:29K epch:5.80 loss:0.058 grdn:0.265 lr:7.6e-05 updt_s:0.675 data_s:0.625 smp/s:49 mem_gb:39.64
116
+ Training: 33%|β–ˆβ–ˆβ–ˆβ–Ž | 20000/60000 [7:24:10<14:40:40, 1.32s/step]INFO 2026-07-14 23:37:01 ot_train.py:625 step:20K smpl:1M ep:29K epch:5.86 loss:0.058 grdn:0.277 lr:7.6e-05 updt_s:0.723 data_s:0.666 smp/s:46 mem_gb:39.63
117
+ Training: 34%|β–ˆβ–ˆβ–ˆβ–Ž | 20200/60000 [7:29:08<17:58:49, 1.63s/step]INFO 2026-07-14 23:41:59 ot_train.py:625 step:20K smpl:1M ep:29K epch:5.92 loss:0.060 grdn:0.285 lr:7.6e-05 updt_s:0.756 data_s:0.732 smp/s:43 mem_gb:39.64
118
+ Training: 34%|β–ˆβ–ˆβ–ˆβ– | 20400/60000 [7:34:25<16:39:59, 1.52s/step]INFO 2026-07-14 23:47:16 ot_train.py:625 step:20K smpl:1M ep:29K epch:5.98 loss:0.056 grdn:0.268 lr:7.5e-05 updt_s:0.799 data_s:0.781 smp/s:41 mem_gb:39.62
119
+ Training: 34%|β–ˆβ–ˆβ–ˆβ– | 20600/60000 [7:39:43<15:34:50, 1.42s/step]INFO 2026-07-14 23:52:34 ot_train.py:625 step:21K smpl:1M ep:30K epch:6.04 loss:0.055 grdn:0.262 lr:7.5e-05 updt_s:0.797 data_s:0.791 smp/s:40 mem_gb:39.64
120
+ Training: 35%|β–ˆβ–ˆβ–ˆβ– | 20800/60000 [7:45:03<18:00:15, 1.65s/step]INFO 2026-07-14 23:57:54 ot_train.py:625 step:21K smpl:1M ep:30K epch:6.10 loss:0.054 grdn:0.258 lr:7.4e-05 updt_s:0.795 data_s:0.801 smp/s:40 mem_gb:39.64
121
+ Training: 35%|β–ˆβ–ˆβ–ˆβ–Œ | 21000/60000 [7:50:23<16:51:45, 1.56s/step]INFO 2026-07-15 00:03:14 ot_train.py:625 step:21K smpl:1M ep:30K epch:6.15 loss:0.056 grdn:0.258 lr:7.4e-05 updt_s:0.790 data_s:0.806 smp/s:40 mem_gb:39.64
122
+ Training: 35%|β–ˆβ–ˆβ–ˆβ–Œ | 21200/60000 [7:55:39<17:37:43, 1.64s/step]INFO 2026-07-15 00:08:30 ot_train.py:625 step:21K smpl:1M ep:31K epch:6.21 loss:0.054 grdn:0.258 lr:7.3e-05 updt_s:0.785 data_s:0.793 smp/s:41 mem_gb:39.64
123
+ Training: 36%|β–ˆβ–ˆβ–ˆβ–Œ | 21400/60000 [8:00:54<17:25:58, 1.63s/step]INFO 2026-07-15 00:13:46 ot_train.py:625 step:21K smpl:1M ep:31K epch:6.27 loss:0.054 grdn:0.258 lr:7.3e-05 updt_s:0.791 data_s:0.784 smp/s:41 mem_gb:39.64
124
+ Training: 36%|β–ˆβ–ˆβ–ˆβ–Œ | 21600/60000 [8:06:01<17:31:34, 1.64s/step]INFO 2026-07-15 00:18:52 ot_train.py:625 step:22K smpl:1M ep:31K epch:6.33 loss:0.055 grdn:0.262 lr:7.2e-05 updt_s:0.773 data_s:0.755 smp/s:42 mem_gb:39.63
125
+ Training: 36%|β–ˆβ–ˆβ–ˆβ–‹ | 21800/60000 [8:11:19<16:43:14, 1.58s/step]INFO 2026-07-15 00:24:10 ot_train.py:625 step:22K smpl:1M ep:31K epch:6.39 loss:0.054 grdn:0.263 lr:7.2e-05 updt_s:0.794 data_s:0.793 smp/s:40 mem_gb:39.64
126
+ Training: 37%|β–ˆβ–ˆβ–ˆβ–‹ | 22000/60000 [8:16:41<17:11:39, 1.63s/step]INFO 2026-07-15 00:29:32 ot_train.py:625 step:22K smpl:1M ep:32K epch:6.45 loss:0.054 grdn:0.269 lr:7.1e-05 updt_s:0.800 data_s:0.806 smp/s:40 mem_gb:39.64
127
+ Training: 37%|β–ˆβ–ˆβ–ˆβ–‹ | 22200/60000 [8:21:59<17:04:40, 1.63s/step]INFO 2026-07-15 00:34:51 ot_train.py:625 step:22K smpl:1M ep:32K epch:6.51 loss:0.054 grdn:0.260 lr:7.1e-05 updt_s:0.791 data_s:0.799 smp/s:40 mem_gb:39.64
128
+ Training: 37%|β–ˆβ–ˆβ–ˆβ–‹ | 22400/60000 [8:27:18<16:23:29, 1.57s/step]INFO 2026-07-15 00:40:09 ot_train.py:625 step:22K smpl:1M ep:32K epch:6.57 loss:0.057 grdn:0.284 lr:7.0e-05 updt_s:0.791 data_s:0.799 smp/s:40 mem_gb:39.64
129
+ Training: 38%|β–ˆβ–ˆβ–ˆβ–Š | 22600/60000 [8:32:28<15:18:06, 1.47s/step]INFO 2026-07-15 00:45:19 ot_train.py:625 step:23K smpl:1M ep:33K epch:6.62 loss:0.053 grdn:0.251 lr:7.0e-05 updt_s:0.774 data_s:0.770 smp/s:41 mem_gb:39.62
130
+ Training: 38%|β–ˆβ–ˆβ–ˆβ–Š | 22800/60000 [8:36:49<11:38:23, 1.13s/step]INFO 2026-07-15 00:49:40 ot_train.py:625 step:23K smpl:1M ep:33K epch:6.68 loss:0.053 grdn:0.265 lr:6.9e-05 updt_s:0.685 data_s:0.619 smp/s:49 mem_gb:39.64
131
+ Training: 38%|β–ˆβ–ˆβ–ˆβ–Š | 23000/60000 [8:41:05<14:19:13, 1.39s/step]INFO 2026-07-15 00:53:56 ot_train.py:625 step:23K smpl:1M ep:33K epch:6.74 loss:0.051 grdn:0.249 lr:6.9e-05 updt_s:0.729 data_s:0.549 smp/s:50 mem_gb:39.64
132
+ Training: 39%|β–ˆβ–ˆβ–ˆβ–Š | 23200/60000 [8:45:41<13:59:08, 1.37s/step]INFO 2026-07-15 00:58:32 ot_train.py:625 step:23K smpl:1M ep:34K epch:6.80 loss:0.052 grdn:0.273 lr:6.8e-05 updt_s:0.806 data_s:0.572 smp/s:46 mem_gb:39.64
133
+ Training: 39%|β–ˆβ–ˆβ–ˆβ–‰ | 23400/60000 [8:50:16<13:59:29, 1.38s/step]INFO 2026-07-15 01:03:07 ot_train.py:625 step:23K smpl:1M ep:34K epch:6.86 loss:0.051 grdn:0.266 lr:6.8e-05 updt_s:0.805 data_s:0.569 smp/s:47 mem_gb:39.64
134
+ Training: 39%|β–ˆβ–ˆβ–ˆβ–‰ | 23600/60000 [8:54:51<13:59:56, 1.38s/step]INFO 2026-07-15 01:07:42 ot_train.py:625 step:24K smpl:2M ep:34K epch:6.92 loss:0.053 grdn:0.266 lr:6.7e-05 updt_s:0.806 data_s:0.564 smp/s:47 mem_gb:39.64
135
+ Training: 40%|β–ˆβ–ˆβ–ˆβ–‰ | 23800/60000 [8:59:25<13:26:36, 1.34s/step]INFO 2026-07-15 01:12:16 ot_train.py:625 step:24K smpl:2M ep:34K epch:6.98 loss:0.051 grdn:0.250 lr:6.7e-05 updt_s:0.801 data_s:0.569 smp/s:47 mem_gb:39.64
136
+ Training: 40%|β–ˆβ–ˆβ–ˆβ–ˆ | 24000/60000 [9:03:54<13:29:52, 1.35s/step]INFO 2026-07-15 01:16:45 ot_train.py:625 step:24K smpl:2M ep:35K epch:7.03 loss:0.050 grdn:0.265 lr:6.6e-05 updt_s:0.765 data_s:0.576 smp/s:48 mem_gb:39.64
137
+ Training: 40%|β–ˆβ–ˆβ–ˆβ–ˆ | 24200/60000 [9:08:27<13:56:27, 1.40s/step]INFO 2026-07-15 01:21:18 ot_train.py:625 step:24K smpl:2M ep:35K epch:7.09 loss:0.052 grdn:0.261 lr:6.6e-05 updt_s:0.799 data_s:0.562 smp/s:47 mem_gb:39.63
138
+ Training: 41%|β–ˆβ–ˆβ–ˆβ–ˆ | 24400/60000 [9:13:08<13:10:45, 1.33s/step]INFO 2026-07-15 01:25:59 ot_train.py:625 step:24K smpl:2M ep:35K epch:7.15 loss:0.051 grdn:0.262 lr:6.5e-05 updt_s:0.834 data_s:0.569 smp/s:46 mem_gb:39.63
139
+ Training: 41%|β–ˆβ–ˆβ–ˆβ–ˆ | 24600/60000 [9:17:45<15:14:13, 1.55s/step]INFO 2026-07-15 01:30:36 ot_train.py:625 step:25K smpl:2M ep:36K epch:7.21 loss:0.052 grdn:0.259 lr:6.5e-05 updt_s:0.825 data_s:0.557 smp/s:46 mem_gb:39.64
140
+ Training: 41%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 24800/60000 [9:22:23<13:03:16, 1.34s/step]INFO 2026-07-15 01:35:14 ot_train.py:625 step:25K smpl:2M ep:36K epch:7.27 loss:0.050 grdn:0.252 lr:6.4e-05 updt_s:0.817 data_s:0.569 smp/s:46 mem_gb:39.64
141
+ Training: 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 25000/60000 [9:26:59<13:23:54, 1.38s/step]INFO 2026-07-15 01:39:50 ot_train.py:625 step:25K smpl:2M ep:36K epch:7.33 loss:0.051 grdn:0.264 lr:6.4e-05 updt_s:0.813 data_s:0.568 smp/s:46 mem_gb:39.62
142
+ Training: 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 25200/60000 [9:31:36<12:46:56, 1.32s/step]INFO 2026-07-15 01:44:27 ot_train.py:625 step:25K smpl:2M ep:36K epch:7.39 loss:0.051 grdn:0.309 lr:6.3e-05 updt_s:0.821 data_s:0.559 smp/s:46 mem_gb:39.64
143
+ Training: 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 25400/60000 [9:36:17<13:28:54, 1.40s/step]INFO 2026-07-15 01:49:08 ot_train.py:625 step:25K smpl:2M ep:37K epch:7.44 loss:0.048 grdn:0.251 lr:6.3e-05 updt_s:0.832 data_s:0.572 smp/s:46 mem_gb:39.64
144
+ Training: 43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 25600/60000 [9:40:57<12:45:29, 1.34s/step]INFO 2026-07-15 01:53:49 ot_train.py:625 step:26K smpl:2M ep:37K epch:7.50 loss:0.050 grdn:0.266 lr:6.2e-05 updt_s:0.843 data_s:0.555 smp/s:46 mem_gb:39.64
145
+ Training: 43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 25800/60000 [9:45:50<14:57:05, 1.57s/step]INFO 2026-07-15 01:58:42 ot_train.py:625 step:26K smpl:2M ep:37K epch:7.56 loss:0.049 grdn:0.269 lr:6.2e-05 updt_s:0.856 data_s:0.607 smp/s:44 mem_gb:39.64
146
+ Training: 43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 26000/60000 [9:51:14<14:39:08, 1.55s/step]INFO 2026-07-15 02:04:05 ot_train.py:625 step:26K smpl:2M ep:38K epch:7.62 loss:0.049 grdn:0.272 lr:6.1e-05 updt_s:0.906 data_s:0.710 smp/s:40 mem_gb:39.64
147
+ Training: 44%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 26200/60000 [9:56:34<15:30:28, 1.65s/step]INFO 2026-07-15 02:09:26 ot_train.py:625 step:26K smpl:2M ep:38K epch:7.68 loss:0.049 grdn:0.263 lr:6.1e-05 updt_s:0.885 data_s:0.714 smp/s:40 mem_gb:39.63
148
+ Training: 44%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 26400/60000 [10:02:02<14:44:53, 1.58s/step]INFO 2026-07-15 02:14:53 ot_train.py:625 step:26K smpl:2M ep:38K epch:7.74 loss:0.049 grdn:0.266 lr:6.0e-05 updt_s:0.889 data_s:0.746 smp/s:39 mem_gb:39.63
149
+ Training: 44%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 26600/60000 [10:07:37<19:16:12, 2.08s/step]INFO 2026-07-15 02:20:28 ot_train.py:625 step:27K smpl:2M ep:38K epch:7.80 loss:0.049 grdn:0.274 lr:6.0e-05 updt_s:0.911 data_s:0.759 smp/s:38 mem_gb:39.64
150
+ Training: 45%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 26800/60000 [10:13:10<15:20:56, 1.66s/step]INFO 2026-07-15 02:26:01 ot_train.py:625 step:27K smpl:2M ep:39K epch:7.85 loss:0.050 grdn:0.270 lr:5.9e-05 updt_s:0.884 data_s:0.779 smp/s:38 mem_gb:39.63
151
+ Training: 45%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27000/60000 [10:18:41<15:53:56, 1.73s/step]INFO 2026-07-15 02:31:32 ot_train.py:625 step:27K smpl:2M ep:39K epch:7.91 loss:0.047 grdn:0.259 lr:5.9e-05 updt_s:0.902 data_s:0.752 smp/s:39 mem_gb:39.63
152
+ Training: 45%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27200/60000 [10:24:13<14:53:00, 1.63s/step]INFO 2026-07-15 02:37:04 ot_train.py:625 step:27K smpl:2M ep:39K epch:7.97 loss:0.047 grdn:0.260 lr:5.8e-05 updt_s:0.876 data_s:0.780 smp/s:39 mem_gb:39.62
153
+ Training: 46%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27400/60000 [10:29:44<15:48:54, 1.75s/step]INFO 2026-07-15 02:42:36 ot_train.py:625 step:27K smpl:2M ep:40K epch:8.03 loss:0.047 grdn:0.273 lr:5.8e-05 updt_s:0.888 data_s:0.766 smp/s:39 mem_gb:39.63
154
+ Training: 46%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27600/60000 [10:35:17<15:05:31, 1.68s/step]INFO 2026-07-15 02:48:08 ot_train.py:625 step:28K smpl:2M ep:40K epch:8.09 loss:0.046 grdn:0.258 lr:5.7e-05 updt_s:0.918 data_s:0.740 smp/s:39 mem_gb:39.64
155
+ Training: 46%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 27800/60000 [10:40:48<14:24:38, 1.61s/step]INFO 2026-07-15 02:53:39 ot_train.py:625 step:28K smpl:2M ep:40K epch:8.15 loss:0.046 grdn:0.258 lr:5.7e-05 updt_s:0.885 data_s:0.768 smp/s:39 mem_gb:39.64
156
+ Training: 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 28000/60000 [10:46:18<14:09:46, 1.59s/step]INFO 2026-07-15 02:59:09 ot_train.py:625 step:28K smpl:2M ep:40K epch:8.21 loss:0.045 grdn:0.259 lr:5.6e-05 updt_s:0.906 data_s:0.742 smp/s:39 mem_gb:39.64
157
+ Training: 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 28200/60000 [10:51:43<15:20:49, 1.74s/step]INFO 2026-07-15 03:04:34 ot_train.py:625 step:28K smpl:2M ep:41K epch:8.26 loss:0.046 grdn:0.271 lr:5.6e-05 updt_s:0.925 data_s:0.696 smp/s:39 mem_gb:39.64
158
+ Training: 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 28400/60000 [10:57:14<15:10:37, 1.73s/step]INFO 2026-07-15 03:10:05 ot_train.py:625 step:28K smpl:2M ep:41K epch:8.32 loss:0.046 grdn:0.280 lr:5.5e-05 updt_s:0.877 data_s:0.776 smp/s:39 mem_gb:39.64
159
+ Training: 48%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 28600/60000 [11:02:46<14:21:58, 1.65s/step]INFO 2026-07-15 03:15:37 ot_train.py:625 step:29K smpl:2M ep:41K epch:8.38 loss:0.046 grdn:0.291 lr:5.5e-05 updt_s:0.909 data_s:0.749 smp/s:39 mem_gb:39.64
160
+ Training: 48%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 28800/60000 [11:08:18<14:46:01, 1.70s/step]INFO 2026-07-15 03:21:09 ot_train.py:625 step:29K smpl:2M ep:42K epch:8.44 loss:0.046 grdn:0.272 lr:5.4e-05 updt_s:0.902 data_s:0.752 smp/s:39 mem_gb:39.64
161
+ Training: 48%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 29000/60000 [11:13:50<14:15:49, 1.66s/step]INFO 2026-07-15 03:26:41 ot_train.py:625 step:29K smpl:2M ep:42K epch:8.50 loss:0.046 grdn:0.273 lr:5.4e-05 updt_s:0.914 data_s:0.746 smp/s:39 mem_gb:39.64
162
+ Training: 49%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 29200/60000 [11:19:24<14:15:17, 1.67s/step]INFO 2026-07-15 03:32:15 ot_train.py:625 step:29K smpl:2M ep:42K epch:8.56 loss:0.046 grdn:0.264 lr:5.3e-05 updt_s:0.879 data_s:0.787 smp/s:38 mem_gb:39.64
163
+ Training: 49%|β–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 29400/60000 [11:24:59<14:02:17, 1.65s/step]INFO 2026-07-15 03:37:50 ot_train.py:625 step:29K smpl:2M ep:42K epch:8.62 loss:0.045 grdn:0.278 lr:5.3e-05 updt_s:0.874 data_s:0.796 smp/s:38 mem_gb:39.62
164
+ Training: 49%|β–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 29600/60000 [11:30:29<13:54:15, 1.65s/step]INFO 2026-07-15 03:43:20 ot_train.py:625 step:30K smpl:2M ep:43K epch:8.68 loss:0.044 grdn:0.278 lr:5.2e-05 updt_s:0.867 data_s:0.779 smp/s:39 mem_gb:39.64
165
+ Training: 50%|β–ˆβ–ˆοΏ½οΏ½οΏ½β–ˆβ–‰ | 29800/60000 [11:36:02<14:42:59, 1.75s/step]INFO 2026-07-15 03:48:53 ot_train.py:625 step:30K smpl:2M ep:43K epch:8.73 loss:0.043 grdn:0.268 lr:5.1e-05 updt_s:0.874 data_s:0.789 smp/s:38 mem_gb:39.64
166
+ Training: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30000/60000 [11:41:34<13:52:00, 1.66s/step]INFO 2026-07-15 03:54:25 ot_train.py:625 step:30K smpl:2M ep:43K epch:8.79 loss:0.043 grdn:0.278 lr:5.1e-05 updt_s:0.878 data_s:0.781 smp/s:39 mem_gb:39.64
167
+ Training: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30200/60000 [11:47:11<13:44:23, 1.66s/step]INFO 2026-07-15 04:00:02 ot_train.py:625 step:30K smpl:2M ep:44K epch:8.85 loss:0.043 grdn:0.276 lr:5.0e-05 updt_s:0.894 data_s:0.786 smp/s:38 mem_gb:39.64
168
+ Training: 51%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30400/60000 [11:52:46<13:39:50, 1.66s/step]INFO 2026-07-15 04:05:37 ot_train.py:625 step:30K smpl:2M ep:44K epch:8.91 loss:0.043 grdn:0.264 lr:5.0e-05 updt_s:0.909 data_s:0.764 smp/s:38 mem_gb:39.64
169
+ Training: 51%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30600/60000 [11:58:20<13:32:44, 1.66s/step]INFO 2026-07-15 04:11:11 ot_train.py:625 step:31K smpl:2M ep:44K epch:8.97 loss:0.042 grdn:0.270 lr:4.9e-05 updt_s:0.873 data_s:0.794 smp/s:38 mem_gb:39.64
170
+ Training: 51%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 30800/60000 [12:03:53<13:26:33, 1.66s/step]INFO 2026-07-15 04:16:45 ot_train.py:625 step:31K smpl:2M ep:45K epch:9.03 loss:0.043 grdn:0.267 lr:4.9e-05 updt_s:0.875 data_s:0.789 smp/s:38 mem_gb:39.64
171
+ Training: 52%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 31000/60000 [12:09:29<13:30:18, 1.68s/step]INFO 2026-07-15 04:22:20 ot_train.py:625 step:31K smpl:2M ep:45K epch:9.09 loss:0.042 grdn:0.270 lr:4.8e-05 updt_s:0.901 data_s:0.774 smp/s:38 mem_gb:39.64
172
+ Training: 52%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 31200/60000 [12:15:01<13:14:58, 1.66s/step]INFO 2026-07-15 04:27:52 ot_train.py:625 step:31K smpl:2M ep:45K epch:9.14 loss:0.042 grdn:0.266 lr:4.8e-05 updt_s:0.872 data_s:0.784 smp/s:39 mem_gb:39.64
173
+ Training: 52%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 31400/60000 [12:20:32<12:46:41, 1.61s/step]INFO 2026-07-15 04:33:23 ot_train.py:625 step:31K smpl:2M ep:45K epch:9.20 loss:0.043 grdn:0.267 lr:4.7e-05 updt_s:0.896 data_s:0.757 smp/s:39 mem_gb:39.64
174
+ Training: 53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 31600/60000 [12:26:06<15:55:47, 2.02s/step]INFO 2026-07-15 04:38:57 ot_train.py:625 step:32K smpl:2M ep:46K epch:9.26 loss:0.042 grdn:0.269 lr:4.7e-05 updt_s:0.900 data_s:0.766 smp/s:38 mem_gb:39.62
175
+ Training: 53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 31800/60000 [12:31:35<11:47:12, 1.50s/step]INFO 2026-07-15 04:44:26 ot_train.py:625 step:32K smpl:2M ep:46K epch:9.32 loss:0.040 grdn:0.260 lr:4.6e-05 updt_s:0.891 data_s:0.751 smp/s:39 mem_gb:39.64
176
+ Training: 53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 32000/60000 [12:37:07<12:48:46, 1.65s/step]INFO 2026-07-15 04:49:58 ot_train.py:625 step:32K smpl:2M ep:46K epch:9.38 loss:0.041 grdn:0.275 lr:4.6e-05 updt_s:0.886 data_s:0.771 smp/s:39 mem_gb:39.64
177
+ Training: 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 32200/60000 [12:42:41<12:45:47, 1.65s/step]INFO 2026-07-15 04:55:32 ot_train.py:625 step:32K smpl:2M ep:47K epch:9.44 loss:0.041 grdn:0.286 lr:4.5e-05 updt_s:0.889 data_s:0.775 smp/s:38 mem_gb:39.64
178
+ Training: 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 32400/60000 [12:48:13<12:43:18, 1.66s/step]INFO 2026-07-15 05:01:04 ot_train.py:625 step:32K smpl:2M ep:47K epch:9.50 loss:0.041 grdn:0.277 lr:4.5e-05 updt_s:0.890 data_s:0.770 smp/s:39 mem_gb:39.63
179
+ Training: 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 32600/60000 [12:53:49<12:41:27, 1.67s/step]INFO 2026-07-15 05:06:40 ot_train.py:625 step:33K smpl:2M ep:47K epch:9.55 loss:0.040 grdn:0.277 lr:4.4e-05 updt_s:0.901 data_s:0.774 smp/s:38 mem_gb:39.64
180
+ Training: 55%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 32800/60000 [12:59:16<12:20:59, 1.63s/step]INFO 2026-07-15 05:12:07 ot_train.py:625 step:33K smpl:2M ep:47K epch:9.61 loss:0.040 grdn:0.271 lr:4.4e-05 updt_s:0.922 data_s:0.713 smp/s:39 mem_gb:39.63
181
+ Training: 55%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33000/60000 [13:04:43<12:14:14, 1.63s/step]INFO 2026-07-15 05:17:34 ot_train.py:625 step:33K smpl:2M ep:48K epch:9.67 loss:0.041 grdn:0.286 lr:4.3e-05 updt_s:0.874 data_s:0.758 smp/s:39 mem_gb:39.63
182
+ Training: 55%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33200/60000 [13:10:12<12:14:11, 1.64s/step]INFO 2026-07-15 05:23:03 ot_train.py:625 step:33K smpl:2M ep:48K epch:9.73 loss:0.040 grdn:0.286 lr:4.3e-05 updt_s:0.899 data_s:0.742 smp/s:39 mem_gb:39.64
183
+ Training: 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33400/60000 [13:15:41<12:09:56, 1.65s/step]INFO 2026-07-15 05:28:32 ot_train.py:625 step:33K smpl:2M ep:48K epch:9.79 loss:0.040 grdn:0.279 lr:4.2e-05 updt_s:0.895 data_s:0.749 smp/s:39 mem_gb:39.64
184
+ Training: 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33600/60000 [13:21:14<12:40:48, 1.73s/step]INFO 2026-07-15 05:34:05 ot_train.py:625 step:34K smpl:2M ep:49K epch:9.85 loss:0.040 grdn:0.287 lr:4.1e-05 updt_s:0.905 data_s:0.754 smp/s:39 mem_gb:39.64
185
+ Training: 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 33800/60000 [13:26:39<11:36:03, 1.59s/step]INFO 2026-07-15 05:39:30 ot_train.py:625 step:34K smpl:2M ep:49K epch:9.91 loss:0.040 grdn:0.281 lr:4.1e-05 updt_s:0.930 data_s:0.691 smp/s:39 mem_gb:39.64
186
+ Training: 57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 34000/60000 [13:32:14<11:58:23, 1.66s/step]INFO 2026-07-15 05:45:05 ot_train.py:625 step:34K smpl:2M ep:49K epch:9.96 loss:0.038 grdn:0.264 lr:4.0e-05 updt_s:0.932 data_s:0.742 smp/s:38 mem_gb:39.62
187
+ Training: 57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 34200/60000 [13:37:48<12:17:56, 1.72s/step]INFO 2026-07-15 05:50:39 ot_train.py:625 step:34K smpl:2M ep:49K epch:10.02 loss:0.037 grdn:0.260 lr:4.0e-05 updt_s:0.891 data_s:0.773 smp/s:38 mem_gb:39.64
188
+ Training: 57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 34400/60000 [13:43:24<12:10:03, 1.71s/step]INFO 2026-07-15 05:56:15 ot_train.py:625 step:34K smpl:2M ep:50K epch:10.08 loss:0.037 grdn:0.274 lr:3.9e-05 updt_s:0.893 data_s:0.786 smp/s:38 mem_gb:39.63
189
+ Training: 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 34600/60000 [13:48:56<11:45:11, 1.67s/step]INFO 2026-07-15 06:01:47 ot_train.py:625 step:35K smpl:2M ep:50K epch:10.14 loss:0.037 grdn:0.273 lr:3.9e-05 updt_s:0.899 data_s:0.756 smp/s:39 mem_gb:39.63
190
+ Training: 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 34800/60000 [13:54:27<11:31:52, 1.65s/step]INFO 2026-07-15 06:07:18 ot_train.py:625 step:35K smpl:2M ep:50K epch:10.20 loss:0.038 grdn:0.269 lr:3.8e-05 updt_s:0.874 data_s:0.780 smp/s:39 mem_gb:39.64
191
+ Training: 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 35000/60000 [13:59:57<11:15:26, 1.62s/step]INFO 2026-07-15 06:12:48 ot_train.py:625 step:35K smpl:2M ep:51K epch:10.26 loss:0.037 grdn:0.277 lr:3.8e-05 updt_s:0.881 data_s:0.763 smp/s:39 mem_gb:39.63
192
+ Training: 59%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 35200/60000 [14:05:28<11:24:17, 1.66s/step]INFO 2026-07-15 06:18:19 ot_train.py:625 step:35K smpl:2M ep:51K epch:10.32 loss:0.037 grdn:0.300 lr:3.7e-05 updt_s:0.896 data_s:0.759 smp/s:39 mem_gb:39.64
193
+ Training: 59%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 35400/60000 [14:11:00<11:26:13, 1.67s/step]INFO 2026-07-15 06:23:51 ot_train.py:625 step:35K smpl:2M ep:51K epch:10.38 loss:0.037 grdn:0.274 lr:3.7e-05 updt_s:0.886 data_s:0.770 smp/s:39 mem_gb:39.64
194
+ Training: 59%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 35600/60000 [14:16:34<11:16:52, 1.66s/step]INFO 2026-07-15 06:29:25 ot_train.py:625 step:36K smpl:2M ep:51K epch:10.43 loss:0.036 grdn:0.276 lr:3.6e-05 updt_s:0.905 data_s:0.763 smp/s:38 mem_gb:39.63
195
+ Training: 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 35800/60000 [14:22:04<11:07:28, 1.65s/step]INFO 2026-07-15 06:34:56 ot_train.py:625 step:36K smpl:2M ep:52K epch:10.49 loss:0.036 grdn:0.273 lr:3.6e-05 updt_s:0.896 data_s:0.751 smp/s:39 mem_gb:39.64
196
+ Training: 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36000/60000 [14:27:41<12:01:42, 1.80s/step]INFO 2026-07-15 06:40:32 ot_train.py:625 step:36K smpl:2M ep:52K epch:10.55 loss:0.037 grdn:0.277 lr:3.5e-05 updt_s:0.890 data_s:0.791 smp/s:38 mem_gb:39.64
197
+ Training: 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36200/60000 [14:33:22<11:10:40, 1.69s/step]INFO 2026-07-15 06:46:13 ot_train.py:625 step:36K smpl:2M ep:52K epch:10.61 loss:0.036 grdn:0.278 lr:3.5e-05 updt_s:0.918 data_s:0.784 smp/s:38 mem_gb:39.62
198
+ Training: 61%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36400/60000 [14:38:57<10:49:25, 1.65s/step]INFO 2026-07-15 06:51:48 ot_train.py:625 step:36K smpl:2M ep:53K epch:10.67 loss:0.035 grdn:0.279 lr:3.4e-05 updt_s:0.911 data_s:0.759 smp/s:38 mem_gb:39.64
199
+ Training: 61%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36600/60000 [14:44:28<10:57:38, 1.69s/step]INFO 2026-07-15 06:57:19 ot_train.py:625 step:37K smpl:2M ep:53K epch:10.73 loss:0.034 grdn:0.267 lr:3.4e-05 updt_s:0.908 data_s:0.745 smp/s:39 mem_gb:39.63
200
+ Training: 61%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 36800/60000 [14:50:01<10:47:55, 1.68s/step]INFO 2026-07-15 07:02:52 ot_train.py:625 step:37K smpl:2M ep:53K epch:10.79 loss:0.034 grdn:0.272 lr:3.3e-05 updt_s:0.887 data_s:0.776 smp/s:38 mem_gb:39.63
201
+ Training: 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 37000/60000 [14:55:40<10:28:31, 1.64s/step]INFO 2026-07-15 07:08:31 ot_train.py:625 step:37K smpl:2M ep:53K epch:10.84 loss:0.034 grdn:0.274 lr:3.3e-05 updt_s:0.899 data_s:0.791 smp/s:38 mem_gb:39.64
202
+ Training: 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 37200/60000 [15:01:12<10:25:14, 1.65s/step]INFO 2026-07-15 07:14:03 ot_train.py:625 step:37K smpl:2M ep:54K epch:10.90 loss:0.034 grdn:0.287 lr:3.2e-05 updt_s:0.881 data_s:0.778 smp/s:39 mem_gb:39.64
203
+ Training: 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 37400/60000 [15:06:48<10:37:13, 1.69s/step]INFO 2026-07-15 07:19:39 ot_train.py:625 step:37K smpl:2M ep:54K epch:10.96 loss:0.035 grdn:0.311 lr:3.2e-05 updt_s:0.895 data_s:0.779 smp/s:38 mem_gb:39.63
204
+ Training: 63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 37600/60000 [15:12:20<10:12:56, 1.64s/step]INFO 2026-07-15 07:25:11 ot_train.py:625 step:38K smpl:2M ep:54K epch:11.02 loss:0.034 grdn:0.274 lr:3.1e-05 updt_s:0.865 data_s:0.795 smp/s:39 mem_gb:39.64
205
+ Training: 63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 37800/60000 [15:17:54<10:06:55, 1.64s/step]INFO 2026-07-15 07:30:45 ot_train.py:625 step:38K smpl:2M ep:55K epch:11.08 loss:0.034 grdn:0.285 lr:3.1e-05 updt_s:0.894 data_s:0.770 smp/s:38 mem_gb:39.64
206
+ Training: 63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 38000/60000 [15:23:32<10:56:34, 1.79s/step]INFO 2026-07-15 07:36:23 ot_train.py:625 step:38K smpl:2M ep:55K epch:11.14 loss:0.034 grdn:0.276 lr:3.0e-05 updt_s:0.907 data_s:0.781 smp/s:38 mem_gb:39.63
207
+ Training: 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 38200/60000 [15:29:03<10:07:18, 1.67s/step]INFO 2026-07-15 07:41:54 ot_train.py:625 step:38K smpl:2M ep:55K epch:11.20 loss:0.034 grdn:0.279 lr:3.0e-05 updt_s:0.898 data_s:0.757 smp/s:39 mem_gb:39.64
208
+ Training: 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 38400/60000 [15:34:41<9:42:31, 1.62s/step]INFO 2026-07-15 07:47:32 ot_train.py:625 step:38K smpl:2M ep:55K epch:11.25 loss:0.034 grdn:0.278 lr:2.9e-05 updt_s:0.893 data_s:0.794 smp/s:38 mem_gb:39.62
209
+ Training: 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 38600/60000 [15:40:13<9:58:51, 1.68s/step]INFO 2026-07-15 07:53:04 ot_train.py:625 step:39K smpl:2M ep:56K epch:11.31 loss:0.031 grdn:0.273 lr:2.9e-05 updt_s:0.878 data_s:0.776 smp/s:39 mem_gb:39.64
210
+ Training: 65%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 38800/60000 [15:45:47<9:58:54, 1.70s/step]INFO 2026-07-15 07:58:38 ot_train.py:625 step:39K smpl:2M ep:56K epch:11.37 loss:0.032 grdn:0.269 lr:2.8e-05 updt_s:0.872 data_s:0.794 smp/s:38 mem_gb:39.64
211
+ Training: 65%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39000/60000 [15:51:21<9:14:08, 1.58s/step]INFO 2026-07-15 08:04:12 ot_train.py:625 step:39K smpl:2M ep:56K epch:11.43 loss:0.032 grdn:0.268 lr:2.8e-05 updt_s:0.876 data_s:0.793 smp/s:38 mem_gb:39.64
212
+ Training: 65%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39200/60000 [15:56:53<10:02:35, 1.74s/step]INFO 2026-07-15 08:09:44 ot_train.py:625 step:39K smpl:3M ep:57K epch:11.49 loss:0.030 grdn:0.278 lr:2.7e-05 updt_s:0.888 data_s:0.768 smp/s:39 mem_gb:39.64
213
+ Training: 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39400/60000 [16:02:25<9:52:44, 1.73s/step]INFO 2026-07-15 08:15:16 ot_train.py:625 step:39K smpl:3M ep:57K epch:11.55 loss:0.032 grdn:0.276 lr:2.7e-05 updt_s:0.897 data_s:0.761 smp/s:39 mem_gb:39.63
214
+ Training: 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39600/60000 [16:08:00<9:22:03, 1.65s/step]INFO 2026-07-15 08:20:51 ot_train.py:625 step:40K smpl:3M ep:57K epch:11.61 loss:0.032 grdn:0.283 lr:2.7e-05 updt_s:0.891 data_s:0.778 smp/s:38 mem_gb:39.64
215
+ Training: 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 39800/60000 [16:13:33<9:25:39, 1.68s/step]INFO 2026-07-15 08:26:24 ot_train.py:625 step:40K smpl:3M ep:58K epch:11.66 loss:0.032 grdn:0.284 lr:2.6e-05 updt_s:0.881 data_s:0.783 smp/s:38 mem_gb:39.64
216
+ Training: 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 40000/60000 [16:19:06<9:08:31, 1.65s/step]INFO 2026-07-15 08:31:57 ot_train.py:625 step:40K smpl:3M ep:58K epch:11.72 loss:0.031 grdn:0.275 lr:2.6e-05 updt_s:0.899 data_s:0.763 smp/s:38 mem_gb:39.64
217
+ Training: 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 40200/60000 [16:24:36<9:11:55, 1.67s/step]INFO 2026-07-15 08:37:27 ot_train.py:625 step:40K smpl:3M ep:58K epch:11.78 loss:0.032 grdn:0.277 lr:2.5e-05 updt_s:0.888 data_s:0.757 smp/s:39 mem_gb:39.64
218
+ Training: 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 40400/60000 [16:30:08<9:18:53, 1.71s/step]INFO 2026-07-15 08:42:59 ot_train.py:625 step:40K smpl:3M ep:58K epch:11.84 loss:0.030 grdn:0.271 lr:2.5e-05 updt_s:0.887 data_s:0.771 smp/s:39 mem_gb:39.64
219
+ Training: 68%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 40600/60000 [16:35:41<9:07:31, 1.69s/step]INFO 2026-07-15 08:48:32 ot_train.py:625 step:41K smpl:3M ep:59K epch:11.90 loss:0.030 grdn:0.292 lr:2.4e-05 updt_s:0.883 data_s:0.777 smp/s:39 mem_gb:39.64
220
+ Training: 68%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 40800/60000 [16:41:13<8:45:21, 1.64s/step]INFO 2026-07-15 08:54:04 ot_train.py:625 step:41K smpl:3M ep:59K epch:11.96 loss:0.029 grdn:0.272 lr:2.4e-05 updt_s:0.898 data_s:0.761 smp/s:39 mem_gb:39.62
221
+ Training: 68%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 41000/60000 [16:46:49<8:40:00, 1.64s/step]INFO 2026-07-15 08:59:40 ot_train.py:625 step:41K smpl:3M ep:59K epch:12.02 loss:0.030 grdn:0.284 lr:2.3e-05 updt_s:0.895 data_s:0.783 smp/s:38 mem_gb:39.64
222
+ Training: 69%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 41200/60000 [16:52:23<8:35:24, 1.64s/step]INFO 2026-07-15 09:05:14 ot_train.py:625 step:41K smpl:3M ep:60K epch:12.08 loss:0.029 grdn:0.289 lr:2.3e-05 updt_s:0.893 data_s:0.770 smp/s:38 mem_gb:39.63
223
+ Training: 69%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 41400/60000 [16:57:54<8:40:07, 1.68s/step]INFO 2026-07-15 09:10:45 ot_train.py:625 step:41K smpl:3M ep:60K epch:12.13 loss:0.029 grdn:0.262 lr:2.2e-05 updt_s:0.888 data_s:0.767 smp/s:39 mem_gb:39.64
224
+ Training: 69%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 41600/60000 [17:03:26<8:20:39, 1.63s/step]INFO 2026-07-15 09:16:17 ot_train.py:625 step:42K smpl:3M ep:60K epch:12.19 loss:0.027 grdn:0.266 lr:2.2e-05 updt_s:0.867 data_s:0.789 smp/s:39 mem_gb:39.64
225
+ Training: 70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 41800/60000 [17:10:48<20:41:28, 4.09s/step]INFO 2026-07-15 09:23:39 ot_train.py:625 step:42K smpl:3M ep:60K epch:12.25 loss:0.029 grdn:0.273 lr:2.2e-05 updt_s:1.412 data_s:0.794 smp/s:29 mem_gb:39.64
226
+ Training: 70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42000/60000 [17:24:24<20:31:05, 4.10s/step]INFO 2026-07-15 09:37:15 ot_train.py:625 step:42K smpl:3M ep:61K epch:12.31 loss:0.029 grdn:0.269 lr:2.1e-05 updt_s:3.228 data_s:0.847 smp/s:16 mem_gb:39.64
227
+ Training: 70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42200/60000 [17:35:54<12:07:31, 2.45s/step]INFO 2026-07-15 09:48:45 ot_train.py:625 step:42K smpl:3M ep:61K epch:12.37 loss:0.029 grdn:0.271 lr:2.1e-05 updt_s:2.618 data_s:0.831 smp/s:19 mem_gb:39.64
228
+ Training: 71%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42400/60000 [17:44:04<11:56:06, 2.44s/step]INFO 2026-07-15 09:56:56 ot_train.py:625 step:42K smpl:3M ep:61K epch:12.43 loss:0.029 grdn:0.279 lr:2.0e-05 updt_s:1.822 data_s:0.627 smp/s:26 mem_gb:39.64
229
+ Training: 71%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42600/60000 [17:55:05<18:12:31, 3.77s/step]INFO 2026-07-15 10:07:56 ot_train.py:625 step:43K smpl:3M ep:62K epch:12.49 loss:0.029 grdn:0.269 lr:2.0e-05 updt_s:2.721 data_s:0.578 smp/s:19 mem_gb:39.64
230
+ Training: 71%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 42800/60000 [18:07:29<17:56:14, 3.75s/step]INFO 2026-07-15 10:20:20 ot_train.py:625 step:43K smpl:3M ep:62K epch:12.54 loss:0.029 grdn:0.267 lr:1.9e-05 updt_s:3.151 data_s:0.568 smp/s:17 mem_gb:39.64
231
+ Training: 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 43000/60000 [18:16:24<12:50:33, 2.72s/step]INFO 2026-07-15 10:29:15 ot_train.py:625 step:43K smpl:3M ep:62K epch:12.60 loss:0.027 grdn:0.270 lr:1.9e-05 updt_s:2.092 data_s:0.579 smp/s:24 mem_gb:39.62
232
+ Training: 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 43200/60000 [18:24:18<11:55:31, 2.56s/step]INFO 2026-07-15 10:37:10 ot_train.py:625 step:43K smpl:3M ep:62K epch:12.66 loss:0.027 grdn:0.273 lr:1.9e-05 updt_s:1.796 data_s:0.575 smp/s:27 mem_gb:39.64
233
+ Training: 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 43400/60000 [18:32:45<12:08:05, 2.63s/step]INFO 2026-07-15 10:45:36 ot_train.py:625 step:43K smpl:3M ep:63K epch:12.72 loss:0.028 grdn:0.274 lr:1.8e-05 updt_s:1.957 data_s:0.572 smp/s:25 mem_gb:39.63
234
+ Training: 73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 43600/60000 [18:41:16<11:03:50, 2.43s/step]INFO 2026-07-15 10:54:07 ot_train.py:625 step:44K smpl:3M ep:63K epch:12.78 loss:0.026 grdn:0.259 lr:1.8e-05 updt_s:1.980 data_s:0.574 smp/s:25 mem_gb:39.64
235
+ Training: 73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 43800/60000 [18:47:18<8:26:27, 1.88s/step]INFO 2026-07-15 11:00:09 ot_train.py:625 step:44K smpl:3M ep:63K epch:12.84 loss:0.027 grdn:0.273 lr:1.7e-05 updt_s:1.213 data_s:0.592 smp/s:35 mem_gb:39.64
236
+ Training: 73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 44000/60000 [18:54:49<12:06:04, 2.72s/step]INFO 2026-07-15 11:07:40 ot_train.py:625 step:44K smpl:3M ep:64K epch:12.90 loss:0.027 grdn:0.257 lr:1.7e-05 updt_s:1.629 data_s:0.624 smp/s:28 mem_gb:39.64
237
+ Training: 74%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 44200/60000 [19:03:46<11:36:50, 2.65s/step]INFO 2026-07-15 11:16:37 ot_train.py:625 step:44K smpl:3M ep:64K epch:12.95 loss:0.027 grdn:0.286 lr:1.7e-05 updt_s:2.081 data_s:0.601 smp/s:24 mem_gb:39.64
238
+ Training: 74%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 44400/60000 [19:11:34<11:23:20, 2.63s/step]INFO 2026-07-15 11:24:25 ot_train.py:625 step:44K smpl:3M ep:64K epch:13.01 loss:0.027 grdn:0.281 lr:1.6e-05 updt_s:1.766 data_s:0.572 smp/s:27 mem_gb:39.64
239
+ Training: 74%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 44600/60000 [19:20:07<11:26:35, 2.68s/step]INFO 2026-07-15 11:32:58 ot_train.py:625 step:45K smpl:3M ep:64K epch:13.07 loss:0.026 grdn:0.264 lr:1.6e-05 updt_s:1.982 data_s:0.579 smp/s:25 mem_gb:39.64
240
+ Training: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 44800/60000 [19:27:32<8:00:15, 1.90s/step]INFO 2026-07-15 11:40:23 ot_train.py:625 step:45K smpl:3M ep:65K epch:13.13 loss:0.026 grdn:0.263 lr:1.5e-05 updt_s:1.652 data_s:0.572 smp/s:29 mem_gb:39.64
241
+ Training: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45000/60000 [19:33:50<5:02:16, 1.21s/step]INFO 2026-07-15 11:46:41 ot_train.py:625 step:45K smpl:3M ep:65K epch:13.19 loss:0.026 grdn:0.264 lr:1.5e-05 updt_s:1.243 data_s:0.641 smp/s:34 mem_gb:39.63
242
+ Training: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45200/60000 [19:42:45<10:54:27, 2.65s/step]INFO 2026-07-15 11:55:36 ot_train.py:625 step:45K smpl:3M ep:65K epch:13.25 loss:0.026 grdn:0.262 lr:1.5e-05 updt_s:2.077 data_s:0.596 smp/s:24 mem_gb:39.62
243
+ Training: 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45400/60000 [19:51:39<11:06:40, 2.74s/step]INFO 2026-07-15 12:04:30 ot_train.py:625 step:45K smpl:3M ep:66K epch:13.31 loss:0.025 grdn:0.251 lr:1.4e-05 updt_s:2.088 data_s:0.582 smp/s:24 mem_gb:39.64
244
+ Training: 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45600/60000 [19:59:38<10:22:59, 2.60s/step]INFO 2026-07-15 12:12:29 ot_train.py:625 step:46K smpl:3M ep:66K epch:13.36 loss:0.026 grdn:0.295 lr:1.4e-05 updt_s:1.812 data_s:0.578 smp/s:27 mem_gb:39.64
245
+ Training: 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 45800/60000 [20:08:05<5:11:53, 1.32s/step]INFO 2026-07-15 12:20:56 ot_train.py:625 step:46K smpl:3M ep:66K epch:13.42 loss:0.025 grdn:0.257 lr:1.4e-05 updt_s:1.964 data_s:0.570 smp/s:25 mem_gb:39.64
246
+ Training: 77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 46000/60000 [20:14:12<7:11:12, 1.85s/step]INFO 2026-07-15 12:27:03 ot_train.py:625 step:46K smpl:3M ep:66K epch:13.48 loss:0.024 grdn:0.257 lr:1.3e-05 updt_s:1.194 data_s:0.637 smp/s:35 mem_gb:39.63
247
+ Training: 77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 46200/60000 [20:22:15<10:13:05, 2.67s/step]INFO 2026-07-15 12:35:06 ot_train.py:625 step:46K smpl:3M ep:67K epch:13.54 loss:0.024 grdn:0.258 lr:1.3e-05 updt_s:1.828 data_s:0.582 smp/s:27 mem_gb:39.64
248
+ Training: 77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 46400/60000 [20:30:14<9:40:57, 2.56s/step]INFO 2026-07-15 12:43:05 ot_train.py:625 step:46K smpl:3M ep:67K epch:13.60 loss:0.024 grdn:0.266 lr:1.3e-05 updt_s:1.821 data_s:0.571 smp/s:27 mem_gb:39.63
249
+ Training: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 46600/60000 [20:37:49<7:04:35, 1.90s/step]INFO 2026-07-15 12:50:40 ot_train.py:625 step:47K smpl:3M ep:67K epch:13.66 loss:0.024 grdn:0.260 lr:1.2e-05 updt_s:1.693 data_s:0.578 smp/s:28 mem_gb:39.64
250
+ Training: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 46800/60000 [20:44:09<7:38:08, 2.08s/step]INFO 2026-07-15 12:57:00 ot_train.py:625 step:47K smpl:3M ep:68K epch:13.72 loss:0.024 grdn:0.256 lr:1.2e-05 updt_s:1.261 data_s:0.638 smp/s:34 mem_gb:39.64
251
+ Training: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 47000/60000 [20:52:10<9:41:08, 2.68s/step]INFO 2026-07-15 13:05:01 ot_train.py:625 step:47K smpl:3M ep:68K epch:13.77 loss:0.024 grdn:0.265 lr:1.1e-05 updt_s:1.825 data_s:0.575 smp/s:27 mem_gb:39.63
252
+ Training: 79%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 47200/60000 [21:00:56<9:30:27, 2.67s/step]INFO 2026-07-15 13:13:47 ot_train.py:625 step:47K smpl:3M ep:68K epch:13.83 loss:0.024 grdn:0.264 lr:1.1e-05 updt_s:2.050 data_s:0.577 smp/s:24 mem_gb:39.63
253
+ Training: 79%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 47400/60000 [21:09:19<4:47:27, 1.37s/step]INFO 2026-07-15 13:22:10 ot_train.py:625 step:47K smpl:3M ep:68K epch:13.89 loss:0.024 grdn:0.276 lr:1.1e-05 updt_s:1.935 data_s:0.580 smp/s:25 mem_gb:39.62
254
+ Training: 79%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 47600/60000 [21:17:27<8:38:29, 2.51s/step]INFO 2026-07-15 13:30:19 ot_train.py:625 step:48K smpl:3M ep:69K epch:13.95 loss:0.023 grdn:0.254 lr:1.0e-05 updt_s:1.870 data_s:0.569 smp/s:26 mem_gb:39.64
255
+ Training: 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 47800/60000 [21:26:57<10:57:31, 3.23s/step]INFO 2026-07-15 13:39:48 ot_train.py:625 step:48K smpl:3M ep:69K epch:14.01 loss:0.023 grdn:0.274 lr:1.0e-05 updt_s:2.158 data_s:0.688 smp/s:22 mem_gb:39.63
256
+ Training: 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48000/60000 [21:35:23<6:19:34, 1.90s/step]INFO 2026-07-15 13:48:14 ot_train.py:625 step:48K smpl:3M ep:69K epch:14.07 loss:0.023 grdn:0.253 lr:9.9e-06 updt_s:1.758 data_s:0.769 smp/s:25 mem_gb:39.63
257
+ Training: 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48200/60000 [21:40:03<3:53:51, 1.19s/step]INFO 2026-07-15 13:52:54 ot_train.py:625 step:48K smpl:3M ep:70K epch:14.13 loss:0.023 grdn:0.259 lr:9.5e-06 updt_s:0.794 data_s:0.604 smp/s:46 mem_gb:39.64
258
+ Training: 81%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48400/60000 [21:44:09<3:56:28, 1.22s/step]INFO 2026-07-15 13:57:00 ot_train.py:625 step:48K smpl:3M ep:70K epch:14.19 loss:0.022 grdn:0.250 lr:9.2e-06 updt_s:0.647 data_s:0.577 smp/s:52 mem_gb:39.63
259
+ Training: 81%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48600/60000 [21:48:11<3:45:58, 1.19s/step]INFO 2026-07-15 14:01:02 ot_train.py:625 step:49K smpl:3M ep:70K epch:14.24 loss:0.023 grdn:0.260 lr:8.9e-06 updt_s:0.639 data_s:0.568 smp/s:53 mem_gb:39.64
260
+ Training: 81%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 48800/60000 [21:52:14<3:41:56, 1.19s/step]INFO 2026-07-15 14:05:05 ot_train.py:625 step:49K smpl:3M ep:71K epch:14.30 loss:0.024 grdn:0.253 lr:8.6e-06 updt_s:0.644 data_s:0.572 smp/s:53 mem_gb:39.63
261
+ Training: 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 49000/60000 [21:56:15<3:32:43, 1.16s/step]INFO 2026-07-15 14:09:06 ot_train.py:625 step:49K smpl:3M ep:71K epch:14.36 loss:0.023 grdn:0.245 lr:8.3e-06 updt_s:0.643 data_s:0.558 smp/s:53 mem_gb:39.63
262
+ Training: 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 49200/60000 [22:00:34<3:41:21, 1.23s/step]INFO 2026-07-15 14:13:25 ot_train.py:625 step:49K smpl:3M ep:71K epch:14.42 loss:0.022 grdn:0.235 lr:8.1e-06 updt_s:0.735 data_s:0.558 smp/s:49 mem_gb:39.63
263
+ Training: 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 49400/60000 [22:04:49<4:14:51, 1.44s/step]INFO 2026-07-15 14:17:40 ot_train.py:625 step:49K smpl:3M ep:71K epch:14.48 loss:0.023 grdn:0.253 lr:7.8e-06 updt_s:0.667 data_s:0.603 smp/s:50 mem_gb:39.64
264
+ Training: 83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 49600/60000 [22:09:17<4:17:15, 1.48s/step]INFO 2026-07-15 14:22:08 ot_train.py:625 step:50K smpl:3M ep:72K epch:14.54 loss:0.022 grdn:0.249 lr:7.5e-06 updt_s:0.689 data_s:0.649 smp/s:48 mem_gb:39.64
265
+ Training: 83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 49800/60000 [22:14:03<3:50:24, 1.36s/step]INFO 2026-07-15 14:26:54 ot_train.py:625 step:50K smpl:3M ep:72K epch:14.60 loss:0.022 grdn:0.252 lr:7.2e-06 updt_s:0.720 data_s:0.706 smp/s:45 mem_gb:39.62
266
+ Training: 83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 50000/60000 [22:18:50<3:54:21, 1.41s/step]INFO 2026-07-15 14:31:41 ot_train.py:625 step:50K smpl:3M ep:72K epch:14.65 loss:0.022 grdn:0.246 lr:6.9e-06 updt_s:0.721 data_s:0.713 smp/s:45 mem_gb:39.64
267
+ INFO 2026-07-15 14:31:41 ot_train.py:670 Checkpoint policy after step 50000
268
+ INFO 2026-07-15 14:32:10 ain_yaml.py:662 Saved config, action-space, and prompt provenance in /home/ext_minje/groot_insight/Abs_6D/Baseline and /home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000
269
+ Training: 83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 50007/60000 [22:19:28<6:15:26, 2.25s/step]
Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/files/requirements.txt ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pyarrow==25.0.0
2
+ importlib_metadata==9.0.0
3
+ wheel==0.47.0
4
+ cloudpickle==3.1.2
5
+ nvidia-cusolver-cu12==11.7.3.90
6
+ multiprocess==0.70.19
7
+ torch==2.11.0+cu128
8
+ nvidia-nvjitlink-cu12==12.8.93
9
+ fsspec==2026.2.0
10
+ pytz==2026.2
11
+ hf-xet==1.5.1
12
+ tokenizers==0.22.2
13
+ opencv-python-headless==4.13.0.92
14
+ cuda-pathfinder==1.2.2
15
+ frozenlist==1.8.0
16
+ smmap==5.0.3
17
+ httpcore==1.0.9
18
+ nvidia-nvshmem-cu12==3.4.5
19
+ nvidia-cuda-nvrtc-cu12==12.8.93
20
+ annotated-types==0.7.0
21
+ platformdirs==4.10.0
22
+ click==8.4.2
23
+ pyyaml-include==1.4.1
24
+ MarkupSafe==3.0.3
25
+ Jinja2==3.1.6
26
+ urllib3==2.7.0
27
+ nvidia-curand-cu12==10.3.9.90
28
+ certifi==2026.6.17
29
+ nvidia-cuda-runtime-cu12==12.8.90
30
+ torchcodec==0.11.1
31
+ toml==0.10.2
32
+ av==15.1.0
33
+ nvidia-cufft-cu12==11.3.3.83
34
+ attrs==26.1.0
35
+ lerobot==0.6.0
36
+ PyYAML==6.0.3
37
+ safetensors==0.8.0
38
+ diffusers==0.35.2
39
+ requests==2.34.2
40
+ nvidia-nvtx-cu12==12.8.90
41
+ charset-normalizer==3.4.9
42
+ networkx==3.6.1
43
+ cmake==4.1.3
44
+ pydantic_core==2.46.4
45
+ timm==1.0.28
46
+ dm-tree==0.1.10
47
+ nvidia-cufile-cu12==1.13.1.3
48
+ termcolor==3.3.0
49
+ multidict==6.7.1
50
+ filelock==3.29.0
51
+ tzdata==2026.3
52
+ aiohappyeyeballs==2.7.1
53
+ httpx==0.28.1
54
+ nvidia-nccl-cu12==2.28.9
55
+ Pygments==2.20.0
56
+ einops==0.8.2
57
+ nvidia-cusparse-cu12==12.5.8.93
58
+ torchvision==0.26.0+cu128
59
+ dill==0.4.1
60
+ GitPython==3.1.51
61
+ nvidia-cuda-cupti-cu12==12.8.90
62
+ decord==0.6.0
63
+ sympy==1.14.0
64
+ cuda-toolkit==12.8.1
65
+ python-dateutil==2.9.0.post0
66
+ shellingham==1.5.4
67
+ propcache==0.5.2
68
+ Farama-Notifications==0.0.6
69
+ pillow==12.2.0
70
+ cuda-bindings==12.9.4
71
+ huggingface_hub==1.23.0
72
+ pandas==2.3.3
73
+ typing-inspect==0.9.0
74
+ packaging==25.0
75
+ rich==15.0.0
76
+ nvidia-cusparselt-cu12==0.7.1
77
+ setuptools==78.1.0
78
+ wrapt==2.2.2
79
+ mypy_extensions==1.1.0
80
+ tqdm==4.68.4
81
+ draccus==0.10.0
82
+ aiosignal==1.4.0
83
+ mpmath==1.3.0
84
+ numpy==2.2.6
85
+ typing_extensions==4.15.0
86
+ psutil==7.2.2
87
+ mdurl==0.1.2
88
+ nvidia-cublas-cu12==12.8.4.1
89
+ absl-py==2.5.0
90
+ gitdb==4.0.12
91
+ transformers==5.5.4
92
+ accelerate==1.14.0
93
+ pydantic==2.13.4
94
+ pip==26.1.2
95
+ markdown-it-py==4.2.0
96
+ typer==0.26.8
97
+ triton==3.6.0
98
+ wandb==0.27.2
99
+ regex==2026.7.10
100
+ yarl==1.24.2
101
+ jsonlines==4.0.0
102
+ mergedeep==1.3.4
103
+ aiohttp==3.14.1
104
+ datasets==4.8.5
105
+ gymnasium==1.3.0
106
+ peft==0.19.1
107
+ zipp==4.1.0
108
+ idna==3.18
109
+ nvidia-cudnn-cu12==9.19.0.56
110
+ h11==0.16.0
111
+ sentry-sdk==2.65.0
112
+ annotated-doc==0.0.4
113
+ protobuf==7.35.1
114
+ typing-inspection==0.4.2
115
+ anyio==4.14.2
116
+ xxhash==3.8.1
117
+ six==1.17.0
Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/files/wandb-metadata.json ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39",
3
+ "python": "CPython 3.12.13",
4
+ "startedAt": "2026-07-14T07:12:40.764055Z",
5
+ "args": [
6
+ "--config_path=/tmp/insight_lerobot_cfg_gtruilkj/baseline.yaml"
7
+ ],
8
+ "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
9
+ "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
10
+ "codePathLocal": "my_scripts/lerobot_train_yaml.py",
11
+ "git": {
12
+ "remote": "git@github.com:paragon7060/clvla.git",
13
+ "commit": "ba1e47317c7f1aa2008714489345f8ee1bc52332"
14
+ },
15
+ "root": "/home/ext_minje/groot_insight/Abs_6D/Baseline",
16
+ "host": "DGX-H200-01",
17
+ "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python",
18
+ "cpu_count": 112,
19
+ "cpu_count_logical": 224,
20
+ "gpu": "NVIDIA H200",
21
+ "gpu_count": 8,
22
+ "disk": {
23
+ "/": {
24
+ "total": "1888425144320",
25
+ "used": "1234837356544"
26
+ }
27
+ },
28
+ "memory": {
29
+ "total": "2164177248256"
30
+ },
31
+ "gpu_nvidia": [
32
+ {
33
+ "name": "NVIDIA H200",
34
+ "memoryTotal": "150754820096",
35
+ "cudaCores": 16896,
36
+ "architecture": "Hopper",
37
+ "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3"
38
+ },
39
+ {
40
+ "name": "NVIDIA H200",
41
+ "memoryTotal": "150754820096",
42
+ "cudaCores": 16896,
43
+ "architecture": "Hopper",
44
+ "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb"
45
+ },
46
+ {
47
+ "name": "NVIDIA H200",
48
+ "memoryTotal": "150754820096",
49
+ "cudaCores": 16896,
50
+ "architecture": "Hopper",
51
+ "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275"
52
+ },
53
+ {
54
+ "name": "NVIDIA H200",
55
+ "memoryTotal": "150754820096",
56
+ "cudaCores": 16896,
57
+ "architecture": "Hopper",
58
+ "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21"
59
+ },
60
+ {
61
+ "name": "NVIDIA H200",
62
+ "memoryTotal": "150754820096",
63
+ "cudaCores": 16896,
64
+ "architecture": "Hopper",
65
+ "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d"
66
+ },
67
+ {
68
+ "name": "NVIDIA H200",
69
+ "memoryTotal": "150754820096",
70
+ "cudaCores": 16896,
71
+ "architecture": "Hopper",
72
+ "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd"
73
+ },
74
+ {
75
+ "name": "NVIDIA H200",
76
+ "memoryTotal": "150754820096",
77
+ "cudaCores": 16896,
78
+ "architecture": "Hopper",
79
+ "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f"
80
+ },
81
+ {
82
+ "name": "NVIDIA H200",
83
+ "memoryTotal": "150754820096",
84
+ "cudaCores": 16896,
85
+ "architecture": "Hopper",
86
+ "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c"
87
+ }
88
+ ],
89
+ "cudaVersion": "12.9",
90
+ "writerId": "njwh3jqwvkleypwcv23lu7svai2ouvzj"
91
+ }
Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug-core.log ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-07-14T16:12:40.8218567+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp144ap2gv/port-201098.txt","pid":201098,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2026-07-14T16:12:40.822647836+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":201098}
3
+ {"time":"2026-07-14T16:12:40.822552298+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-201098-201620-3722653362/socket","Net":"unix"}}
4
+ {"time":"2026-07-14T16:12:40.992669022+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
+ {"time":"2026-07-14T16:12:41.001152537+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"1jmjb68j","id":"1(@)"}
6
+ {"time":"2026-07-14T16:12:41.620084809+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"1jmjb68j","id":"1(@)"}
7
+ {"time":"2026-07-14T16:12:47.144909049+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"n7zozk137zt4"}
8
+ {"time":"2026-07-15T14:32:24.049418672+09:00","level":"INFO","msg":"server: parent process exited, terminating service process"}
Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-07-14 16:12:40,765 INFO MainThread:201098 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
2
+ 2026-07-14 16:12:40,765 INFO MainThread:201098 [wandb_setup.py:_flush():81] Configure stats pid to 201098
3
+ 2026-07-14 16:12:40,765 INFO MainThread:201098 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-07-14 16:12:40,765 INFO MainThread:201098 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug.log
5
+ 2026-07-14 16:12:40,765 INFO MainThread:201098 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/logs/debug-internal.log
6
+ 2026-07-14 16:12:40,765 INFO MainThread:201098 [wandb_init.py:init():766] calling init triggers
7
+ 2026-07-14 16:12:40,765 INFO MainThread:201098 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
8
+ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/Abs_6D/Baseline', 'job_name': 'INSIGHT_6D_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
9
+ 2026-07-14 16:12:40,766 INFO MainThread:201098 [wandb_init.py:init():814] starting backend
10
+ 2026-07-14 16:12:40,992 INFO MainThread:201098 [wandb_init.py:init():829] sending inform_init request
11
+ 2026-07-14 16:12:41,621 INFO MainThread:201098 [wandb_init.py:init():834] backend started and connected
12
+ 2026-07-14 16:12:41,623 INFO MainThread:201098 [wandb_init.py:init():904] updated telemetry
13
+ 2026-07-14 16:12:41,626 INFO MainThread:201098 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
14
+ 2026-07-14 16:12:42,075 INFO MainThread:201098 [wandb_init.py:init():972] starting run threads in backend
15
+ 2026-07-14 16:12:42,140 INFO MainThread:201098 [wandb_run.py:_console_start():2523] atexit reg
16
+ 2026-07-14 16:12:42,140 INFO MainThread:201098 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
+ 2026-07-14 16:12:42,140 INFO MainThread:201098 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
+ 2026-07-14 16:12:42,141 INFO MainThread:201098 [wandb_run.py:_redirect():2465] Redirects installed.
19
+ 2026-07-14 16:12:42,144 INFO MainThread:201098 [wandb_init.py:init():1010] run started, returning control to user process
Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/config.yaml ADDED
@@ -0,0 +1,351 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.27.2
4
+ e:
5
+ c8d5ybgsruh4m5h3ie5jd18btesa3xht:
6
+ args:
7
+ - --config_path=/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model/train_config.json
8
+ - --resume=true
9
+ codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
10
+ codePathLocal: my_scripts/lerobot_train_yaml.py
11
+ cpu_count: 112
12
+ cpu_count_logical: 224
13
+ cudaVersion: "12.9"
14
+ disk:
15
+ /:
16
+ total: "1888425144320"
17
+ used: "1297062400000"
18
+ executable: /home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python
19
+ git:
20
+ commit: 22b2faefeaab966ca2dc3688fe6a77f3bcd8de32
21
+ remote: git@github.com:paragon7060/clvla.git
22
+ gpu: NVIDIA H200
23
+ gpu_count: 8
24
+ gpu_nvidia:
25
+ - architecture: Hopper
26
+ cudaCores: 16896
27
+ memoryTotal: "150754820096"
28
+ name: NVIDIA H200
29
+ uuid: GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3
30
+ - architecture: Hopper
31
+ cudaCores: 16896
32
+ memoryTotal: "150754820096"
33
+ name: NVIDIA H200
34
+ uuid: GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb
35
+ - architecture: Hopper
36
+ cudaCores: 16896
37
+ memoryTotal: "150754820096"
38
+ name: NVIDIA H200
39
+ uuid: GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275
40
+ - architecture: Hopper
41
+ cudaCores: 16896
42
+ memoryTotal: "150754820096"
43
+ name: NVIDIA H200
44
+ uuid: GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21
45
+ - architecture: Hopper
46
+ cudaCores: 16896
47
+ memoryTotal: "150754820096"
48
+ name: NVIDIA H200
49
+ uuid: GPU-a91176db-55f2-b284-bbbf-56626596947d
50
+ - architecture: Hopper
51
+ cudaCores: 16896
52
+ memoryTotal: "150754820096"
53
+ name: NVIDIA H200
54
+ uuid: GPU-922ff6d5-a948-205a-0426-dbee495fc3dd
55
+ - architecture: Hopper
56
+ cudaCores: 16896
57
+ memoryTotal: "150754820096"
58
+ name: NVIDIA H200
59
+ uuid: GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f
60
+ - architecture: Hopper
61
+ cudaCores: 16896
62
+ memoryTotal: "150754820096"
63
+ name: NVIDIA H200
64
+ uuid: GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c
65
+ host: DGX-H200-01
66
+ memory:
67
+ total: "2164177248256"
68
+ os: Linux-6.8.0-107-generic-x86_64-with-glibc2.39
69
+ program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
70
+ python: CPython 3.12.13
71
+ root: /home/ext_minje/groot_insight/Abs_6D/Baseline
72
+ startedAt: "2026-07-15T05:32:50.279073Z"
73
+ writerId: c8d5ybgsruh4m5h3ie5jd18btesa3xht
74
+ m: []
75
+ python_version: 3.12.13
76
+ t:
77
+ "1":
78
+ - 1
79
+ - 11
80
+ - 41
81
+ - 49
82
+ - 51
83
+ - 71
84
+ - 83
85
+ - 98
86
+ "2":
87
+ - 1
88
+ - 11
89
+ - 41
90
+ - 49
91
+ - 51
92
+ - 71
93
+ - 83
94
+ - 98
95
+ "3":
96
+ - 5
97
+ - 13
98
+ - 14
99
+ - 15
100
+ - 16
101
+ - 61
102
+ - 62
103
+ "4": 3.12.13
104
+ "5": 0.27.2
105
+ "6": 5.5.4
106
+ "10":
107
+ - 21
108
+ "12": 0.27.2
109
+ "13": linux-x86_64
110
+ batch_size:
111
+ value: 64
112
+ checkpoint_path:
113
+ value: /home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000
114
+ cudnn_deterministic:
115
+ value: false
116
+ dataset:
117
+ value:
118
+ depth_output_unit: mm
119
+ episodes: null
120
+ eval_split: 0
121
+ image_transforms:
122
+ enable: false
123
+ max_num_transforms: 3
124
+ random_order: false
125
+ tfs:
126
+ affine:
127
+ kwargs:
128
+ degrees:
129
+ - -5
130
+ - 5
131
+ translate:
132
+ - 0.05
133
+ - 0.05
134
+ type: RandomAffine
135
+ weight: 1
136
+ brightness:
137
+ kwargs:
138
+ brightness:
139
+ - 0.8
140
+ - 1.2
141
+ type: ColorJitter
142
+ weight: 1
143
+ contrast:
144
+ kwargs:
145
+ contrast:
146
+ - 0.8
147
+ - 1.2
148
+ type: ColorJitter
149
+ weight: 1
150
+ hue:
151
+ kwargs:
152
+ hue:
153
+ - -0.05
154
+ - 0.05
155
+ type: ColorJitter
156
+ weight: 1
157
+ saturation:
158
+ kwargs:
159
+ saturation:
160
+ - 0.5
161
+ - 1.5
162
+ type: ColorJitter
163
+ weight: 1
164
+ sharpness:
165
+ kwargs:
166
+ sharpness:
167
+ - 0.5
168
+ - 1.5
169
+ type: SharpnessJitter
170
+ weight: 1
171
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
172
+ return_uint8: false
173
+ revision: null
174
+ root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
175
+ streaming: false
176
+ use_imagenet_stats: true
177
+ video_backend: torchcodec
178
+ env:
179
+ value: null
180
+ env_eval_freq:
181
+ value: 20000
182
+ eval:
183
+ value:
184
+ batch_size: 50
185
+ n_episodes: 50
186
+ recording: false
187
+ recording_private: false
188
+ recording_repo_id: null
189
+ use_async_envs: true
190
+ eval_steps:
191
+ value: 0
192
+ job:
193
+ value:
194
+ detach: false
195
+ image: huggingface/lerobot-gpu:latest
196
+ tags: []
197
+ target: null
198
+ timeout: 2d
199
+ job_name:
200
+ value: INSIGHT_6D_baseline
201
+ log_freq:
202
+ value: 200
203
+ max_eval_samples:
204
+ value: 0
205
+ num_workers:
206
+ value: 4
207
+ optimizer:
208
+ value:
209
+ betas:
210
+ - 0.9
211
+ - 0.999
212
+ eps: 1e-08
213
+ grad_clip_norm: 1
214
+ lr: 0.0001
215
+ type: adamw
216
+ weight_decay: 1e-05
217
+ output_dir:
218
+ value: /home/ext_minje/groot_insight/Abs_6D/Baseline
219
+ peft:
220
+ value: null
221
+ persistent_workers:
222
+ value: true
223
+ policy:
224
+ value:
225
+ action_decode_transform: null
226
+ balance_dataset_weights: true
227
+ balance_trajectory_weights: true
228
+ base_model_path: nvidia/GR00T-N1.7-3B
229
+ batch_size: 32
230
+ chunk_size: 16
231
+ dataloader_num_workers: 8
232
+ dataset_paths: null
233
+ device: cuda
234
+ embodiment_tag: new_embodiment
235
+ image_size:
236
+ - 256
237
+ - 256
238
+ input_features:
239
+ observation.images.guide:
240
+ shape:
241
+ - 3
242
+ - 224
243
+ - 224
244
+ type: VISUAL
245
+ observation.images.right_shoulder:
246
+ shape:
247
+ - 3
248
+ - 224
249
+ - 224
250
+ type: VISUAL
251
+ observation.images.wrist:
252
+ shape:
253
+ - 3
254
+ - 224
255
+ - 224
256
+ type: VISUAL
257
+ observation.state:
258
+ shape:
259
+ - 16
260
+ type: STATE
261
+ license: null
262
+ lora_alpha: 16
263
+ lora_dropout: 0.1
264
+ lora_full_model: false
265
+ lora_rank: 0
266
+ max_action_dim: 132
267
+ max_state_dim: 132
268
+ max_steps: 10000
269
+ model_params_fp32: true
270
+ n_action_steps: 16
271
+ n_obs_steps: 1
272
+ normalization_mapping:
273
+ ACTION: IDENTITY
274
+ STATE: IDENTITY
275
+ VISUAL: IDENTITY
276
+ num_inference_timesteps: null
277
+ optimizer_betas:
278
+ - 0.9
279
+ - 0.999
280
+ optimizer_eps: 1e-08
281
+ optimizer_lr: 0.0001
282
+ optimizer_weight_decay: 1e-05
283
+ output_dir: ./tmp/gr00t
284
+ output_features:
285
+ action:
286
+ shape:
287
+ - 10
288
+ type: ACTION
289
+ pretrained_path: /home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model
290
+ pretrained_revision: null
291
+ private: null
292
+ push_to_hub: false
293
+ relative_exclude_joints: []
294
+ repo_id: null
295
+ report_to: wandb
296
+ resume: false
297
+ rtc_ramp_rate: null
298
+ save_steps: 1000
299
+ tags: null
300
+ tokenizer_assets_repo: null
301
+ tune_diffusion_model: true
302
+ tune_llm: false
303
+ tune_projector: true
304
+ tune_top_llm_layers: 0
305
+ tune_visual: false
306
+ tune_vlln: true
307
+ type: groot
308
+ use_amp: false
309
+ use_bf16: true
310
+ use_flash_attention: false
311
+ use_peft: false
312
+ use_relative_actions: false
313
+ video_backend: decord
314
+ warmup_ratio: 0.05
315
+ prefetch_factor:
316
+ value: 4
317
+ resume:
318
+ value: true
319
+ reward_model:
320
+ value: null
321
+ sample_weighting:
322
+ value: null
323
+ save_checkpoint:
324
+ value: true
325
+ save_checkpoint_to_hub:
326
+ value: false
327
+ save_freq:
328
+ value: 20000
329
+ scheduler:
330
+ value:
331
+ name: cosine
332
+ num_warmup_steps: 500
333
+ type: diffuser
334
+ seed:
335
+ value: 42
336
+ steps:
337
+ value: 60000
338
+ tolerance_s:
339
+ value: 0.0001
340
+ use_policy_training_preset:
341
+ value: true
342
+ wandb:
343
+ value:
344
+ add_tags: true
345
+ disable_artifact: true
346
+ enable: true
347
+ entity: null
348
+ mode: null
349
+ notes: null
350
+ project: lerobot
351
+ run_id: 1jmjb68j
Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/output.log ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO 2026-07-15 14:32:51 db_utils.py:121 Logs will be synced with wandb.
2
+ INFO 2026-07-15 14:32:51 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/1jmjb68j
3
+ INFO 2026-07-15 14:32:51 ot_train.py:298 Creating dataset
4
+ INFO 2026-07-15 14:32:54 ot_train.py:332 Creating policy
5
+ INFO 2026-07-15 14:32:54 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model
6
+ INFO 2026-07-15 14:32:54 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict...
7
+ Fetching 27 files: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 27/27 [00:00<00:00, 1554.62it/s]
8
+ `torch_dtype` is deprecated! Use `dtype` instead!
9
+ Loading weights: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1031/1031 [00:00<00:00, 2321.50it/s]
10
+ Loading weights from local directory
11
+ INFO 2026-07-15 14:33:09 ot_train.py:405 Creating optimizer and scheduler
12
+ INFO 2026-07-15 14:33:12 ot_train.py:437 Output dir: /home/ext_minje/groot_insight/Abs_6D/Baseline
13
+ INFO 2026-07-15 14:33:12 ot_train.py:444 cfg.steps=60000 (60K)
14
+ INFO 2026-07-15 14:33:12 ot_train.py:445 dataset.num_frames=218367 (218K)
15
+ INFO 2026-07-15 14:33:12 ot_train.py:446 dataset.num_episodes=4930
16
+ INFO 2026-07-15 14:33:12 ot_train.py:449 Effective batch size: 64 x 1 = 64
17
+ INFO 2026-07-15 14:33:12 ot_train.py:450 num_learnable_params=1620515968 (2B)
18
+ INFO 2026-07-15 14:33:12 ot_train.py:451 num_total_params=3144016000 (3B)
19
+ INFO 2026-07-15 14:33:12 ot_train.py:492 Resuming data order at epoch 22, sample 22144
20
+ Training: 0%| | 0/10000 [00:00<?, ?step/s]INFO 2026-07-15 14:33:12 ot_train.py:602 Start offline training on a fixed dataset, with effective batch size: 64
21
+ Training: 2%|▏ | 200/10000 [05:24<4:14:48, 1.56s/step]INFO 2026-07-15 14:38:37 ot_train.py:646 step:50K smpl:3M ep:73K epch:14.71 loss:0.021 grdn:0.236 lr:6.7e-06 updt_s:0.924 data_s:0.694 smp/s:40 mem_gb:39.59
22
+ Training: 4%|▍ | 400/10000 [10:52<4:26:31, 1.67s/step]INFO 2026-07-15 14:44:05 ot_train.py:646 step:50K smpl:3M ep:73K epch:14.77 loss:0.021 grdn:0.247 lr:6.4e-06 updt_s:0.915 data_s:0.723 smp/s:39 mem_gb:39.59
23
+ Training: 6%|β–Œ | 600/10000 [16:24<4:12:28, 1.61s/step]INFO 2026-07-15 14:49:37 ot_train.py:646 step:51K smpl:3M ep:73K epch:14.83 loss:0.022 grdn:0.230 lr:6.2e-06 updt_s:0.904 data_s:0.756 smp/s:39 mem_gb:39.59
24
+ Training: 8%|β–Š | 800/10000 [21:58<4:17:42, 1.68s/step]INFO 2026-07-15 14:55:10 ot_train.py:646 step:51K smpl:3M ep:73K epch:14.89 loss:0.022 grdn:0.258 lr:5.9e-06 updt_s:0.879 data_s:0.784 smp/s:38 mem_gb:39.59
25
+ Training: 10%|β–ˆ | 1000/10000 [27:30<4:11:02, 1.67s/step]INFO 2026-07-15 15:00:43 ot_train.py:646 step:51K smpl:3M ep:74K epch:14.95 loss:0.021 grdn:0.242 lr:5.7e-06 updt_s:0.879 data_s:0.783 smp/s:39 mem_gb:39.59
26
+ Training: 12%|β–ˆβ– | 1200/10000 [33:03<4:00:36, 1.64s/step]INFO 2026-07-15 15:06:16 ot_train.py:646 step:51K smpl:3M ep:74K epch:15.01 loss:0.021 grdn:0.248 lr:5.4e-06 updt_s:0.889 data_s:0.772 smp/s:39 mem_gb:39.59
27
+ Training: 14%|β–ˆβ– | 1400/10000 [38:35<4:03:11, 1.70s/step]INFO 2026-07-15 15:11:48 ot_train.py:646 step:51K smpl:3M ep:74K epch:15.06 loss:0.021 grdn:0.238 lr:5.2e-06 updt_s:0.900 data_s:0.759 smp/s:39 mem_gb:39.59
28
+ Training: 16%|β–ˆβ–Œ | 1600/10000 [44:10<3:54:05, 1.67s/step]INFO 2026-07-15 15:17:23 ot_train.py:646 step:52K smpl:3M ep:75K epch:15.12 loss:0.021 grdn:0.243 lr:5.0e-06 updt_s:0.892 data_s:0.779 smp/s:38 mem_gb:39.59
29
+ Training: 18%|β–ˆβ–Š | 1800/10000 [49:42<3:48:52, 1.67s/step]INFO 2026-07-15 15:22:55 ot_train.py:646 step:52K smpl:3M ep:75K epch:15.18 loss:0.022 grdn:0.237 lr:4.7e-06 updt_s:0.887 data_s:0.771 smp/s:39 mem_gb:39.58
30
+ Training: 20%|β–ˆβ–ˆ | 2000/10000 [55:12<3:33:15, 1.60s/step]INFO 2026-07-15 15:28:25 ot_train.py:646 step:52K smpl:3M ep:75K epch:15.24 loss:0.020 grdn:0.244 lr:4.5e-06 updt_s:0.955 data_s:0.691 smp/s:39 mem_gb:39.57
31
+ Training: 22%|β–ˆβ–ˆβ– | 2200/10000 [1:00:44<3:17:56, 1.52s/step]INFO 2026-07-15 15:33:57 ot_train.py:646 step:52K smpl:3M ep:75K epch:15.30 loss:0.021 grdn:0.240 lr:4.3e-06 updt_s:0.918 data_s:0.740 smp/s:39 mem_gb:39.59
32
+ Training: 24%|β–ˆβ–ˆβ– | 2400/10000 [1:06:15<3:27:54, 1.64s/step]INFO 2026-07-15 15:39:27 ot_train.py:646 step:52K smpl:3M ep:76K epch:15.36 loss:0.021 grdn:0.236 lr:4.1e-06 updt_s:0.895 data_s:0.754 smp/s:39 mem_gb:39.59
33
+ Training: 26%|β–ˆβ–ˆβ–Œ | 2600/10000 [1:11:46<3:26:49, 1.68s/step]INFO 2026-07-15 15:44:59 ot_train.py:646 step:53K smpl:3M ep:76K epch:15.42 loss:0.020 grdn:0.243 lr:3.9e-06 updt_s:0.905 data_s:0.750 smp/s:39 mem_gb:39.59
34
+ Training: 28%|β–ˆβ–ˆβ–Š | 2800/10000 [1:17:20<3:10:47, 1.59s/step]INFO 2026-07-15 15:50:33 ot_train.py:646 step:53K smpl:3M ep:76K epch:15.47 loss:0.021 grdn:0.231 lr:3.7e-06 updt_s:0.921 data_s:0.745 smp/s:38 mem_gb:39.59
35
+ Training: 30%|β–ˆβ–ˆβ–ˆ | 3000/10000 [1:22:42<3:09:38, 1.63s/step]INFO 2026-07-15 15:55:55 ot_train.py:646 step:53K smpl:3M ep:77K epch:15.53 loss:0.020 grdn:0.237 lr:3.5e-06 updt_s:0.994 data_s:0.614 smp/s:40 mem_gb:39.59
36
+ Training: 32%|β–ˆβ–ˆβ–ˆβ– | 3200/10000 [1:28:06<3:08:09, 1.66s/step]INFO 2026-07-15 16:01:19 ot_train.py:646 step:53K smpl:3M ep:77K epch:15.59 loss:0.021 grdn:0.246 lr:3.3e-06 updt_s:1.000 data_s:0.618 smp/s:40 mem_gb:39.59
37
+ Training: 34%|β–ˆβ–ˆβ–ˆβ– | 3400/10000 [1:33:30<3:07:51, 1.71s/step]INFO 2026-07-15 16:06:43 ot_train.py:646 step:53K smpl:3M ep:77K epch:15.65 loss:0.020 grdn:0.235 lr:3.1e-06 updt_s:1.008 data_s:0.608 smp/s:40 mem_gb:39.59
38
+ Training: 36%|β–ˆβ–ˆβ–ˆβ–Œ | 3600/10000 [1:38:57<2:58:53, 1.68s/step]INFO 2026-07-15 16:12:10 ot_train.py:646 step:54K smpl:3M ep:77K epch:15.71 loss:0.020 grdn:0.250 lr:2.9e-06 updt_s:1.015 data_s:0.621 smp/s:39 mem_gb:39.59
39
+ Training: 38%|β–ˆβ–ˆβ–ˆβ–Š | 3800/10000 [1:44:17<2:53:33, 1.68s/step]INFO 2026-07-15 16:17:30 ot_train.py:646 step:54K smpl:3M ep:78K epch:15.77 loss:0.020 grdn:0.244 lr:2.7e-06 updt_s:1.002 data_s:0.594 smp/s:40 mem_gb:39.59
40
+ Training: 40%|β–ˆβ–ˆβ–ˆβ–ˆ | 4000/10000 [1:49:34<2:32:27, 1.52s/step]INFO 2026-07-15 16:22:47 ot_train.py:646 step:54K smpl:3M ep:78K epch:15.83 loss:0.021 grdn:0.239 lr:2.6e-06 updt_s:0.994 data_s:0.588 smp/s:40 mem_gb:39.59
41
+ Training: 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 4200/10000 [1:54:56<2:34:18, 1.60s/step]INFO 2026-07-15 16:28:09 ot_train.py:646 step:54K smpl:3M ep:78K epch:15.89 loss:0.020 grdn:0.230 lr:2.4e-06 updt_s:0.985 data_s:0.623 smp/s:40 mem_gb:39.58
42
+ Training: 44%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 4400/10000 [2:00:31<2:35:38, 1.67s/step]INFO 2026-07-15 16:33:44 ot_train.py:646 step:54K smpl:3M ep:79K epch:15.94 loss:0.020 grdn:0.227 lr:2.2e-06 updt_s:0.913 data_s:0.757 smp/s:38 mem_gb:39.59
43
+ Training: 46%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 4600/10000 [2:06:01<2:33:49, 1.71s/step]INFO 2026-07-15 16:39:14 ot_train.py:646 step:55K smpl:3M ep:79K epch:16.00 loss:0.020 grdn:0.230 lr:2.1e-06 updt_s:0.903 data_s:0.746 smp/s:39 mem_gb:39.59
44
+ Training: 48%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 4800/10000 [2:11:35<2:21:30, 1.63s/step]INFO 2026-07-15 16:44:48 ot_train.py:646 step:55K smpl:4M ep:79K epch:16.06 loss:0.020 grdn:0.234 lr:1.9e-06 updt_s:0.928 data_s:0.740 smp/s:38 mem_gb:39.59
45
+ Training: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 5000/10000 [2:17:09<2:18:50, 1.67s/step]INFO 2026-07-15 16:50:22 ot_train.py:646 step:55K smpl:4M ep:79K epch:16.12 loss:0.019 grdn:0.238 lr:1.8e-06 updt_s:0.920 data_s:0.749 smp/s:38 mem_gb:39.59
46
+ Training: 52%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 5200/10000 [2:22:43<2:19:05, 1.74s/step]INFO 2026-07-15 16:55:56 ot_train.py:646 step:55K smpl:4M ep:80K epch:16.18 loss:0.020 grdn:0.234 lr:1.7e-06 updt_s:0.891 data_s:0.774 smp/s:38 mem_gb:39.59
47
+ Training: 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 5400/10000 [2:28:15<2:09:20, 1.69s/step]INFO 2026-07-15 17:01:28 ot_train.py:646 step:55K smpl:4M ep:80K epch:16.24 loss:0.020 grdn:0.237 lr:1.5e-06 updt_s:0.890 data_s:0.771 smp/s:39 mem_gb:39.59
48
+ Training: 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 5600/10000 [2:33:49<2:00:51, 1.65s/step]INFO 2026-07-15 17:07:02 ot_train.py:646 step:56K smpl:4M ep:80K epch:16.30 loss:0.020 grdn:0.228 lr:1.4e-06 updt_s:0.900 data_s:0.764 smp/s:38 mem_gb:39.59
49
+ Training: 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 5800/10000 [2:39:22<1:56:06, 1.66s/step]INFO 2026-07-15 17:12:35 ot_train.py:646 step:56K smpl:4M ep:81K epch:16.35 loss:0.020 grdn:0.230 lr:1.3e-06 updt_s:0.921 data_s:0.744 smp/s:38 mem_gb:39.59
50
+ Training: 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 6000/10000 [2:45:01<1:53:35, 1.70s/step]INFO 2026-07-15 17:18:14 ot_train.py:646 step:56K smpl:4M ep:81K epch:16.41 loss:0.020 grdn:0.233 lr:1.2e-06 updt_s:0.928 data_s:0.763 smp/s:38 mem_gb:39.59
51
+ Training: 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 6200/10000 [2:50:40<1:42:26, 1.62s/step]INFO 2026-07-15 17:23:53 ot_train.py:646 step:56K smpl:4M ep:81K epch:16.47 loss:0.020 grdn:0.234 lr:1.1e-06 updt_s:0.905 data_s:0.789 smp/s:38 mem_gb:39.59
52
+ Training: 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 6400/10000 [2:56:17<2:00:10, 2.00s/step]INFO 2026-07-15 17:29:30 ot_train.py:646 step:56K smpl:4M ep:81K epch:16.53 loss:0.020 grdn:0.230 lr:9.5e-07 updt_s:0.897 data_s:0.785 smp/s:38 mem_gb:39.59
53
+ Training: 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 6600/10000 [3:01:57<1:35:11, 1.68s/step]INFO 2026-07-15 17:35:10 ot_train.py:646 step:57K smpl:4M ep:82K epch:16.59 loss:0.020 grdn:0.236 lr:8.5e-07 updt_s:0.917 data_s:0.782 smp/s:38 mem_gb:39.58
54
+ Training: 68%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 6800/10000 [3:07:35<1:31:24, 1.71s/step]INFO 2026-07-15 17:40:48 ot_train.py:646 step:57K smpl:4M ep:82K epch:16.65 loss:0.019 grdn:0.232 lr:7.6e-07 updt_s:0.954 data_s:0.731 smp/s:38 mem_gb:39.59
55
+ Training: 70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 7000/10000 [3:13:08<1:25:46, 1.72s/step]INFO 2026-07-15 17:46:21 ot_train.py:646 step:57K smpl:4M ep:82K epch:16.71 loss:0.019 grdn:0.221 lr:6.7e-07 updt_s:0.890 data_s:0.772 smp/s:38 mem_gb:39.59
56
+ Training: 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 7200/10000 [3:18:42<1:18:00, 1.67s/step]INFO 2026-07-15 17:51:55 ot_train.py:646 step:57K smpl:4M ep:83K epch:16.76 loss:0.020 grdn:0.244 lr:5.9e-07 updt_s:0.898 data_s:0.771 smp/s:38 mem_gb:39.59
57
+ Training: 74%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆοΏ½οΏ½ | 7400/10000 [3:24:15<1:11:50, 1.66s/step]INFO 2026-07-15 17:57:28 ot_train.py:646 step:57K smpl:4M ep:83K epch:16.82 loss:0.019 grdn:0.220 lr:5.1e-07 updt_s:0.914 data_s:0.748 smp/s:39 mem_gb:39.59
58
+ Training: 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 7600/10000 [3:29:49<1:05:33, 1.64s/step]INFO 2026-07-15 18:03:02 ot_train.py:646 step:58K smpl:4M ep:83K epch:16.88 loss:0.019 grdn:0.225 lr:4.4e-07 updt_s:0.899 data_s:0.770 smp/s:38 mem_gb:39.59
59
+ Training: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 7800/10000 [3:35:24<1:00:16, 1.64s/step]INFO 2026-07-15 18:08:37 ot_train.py:646 step:58K smpl:4M ep:84K epch:16.94 loss:0.019 grdn:0.230 lr:3.7e-07 updt_s:0.901 data_s:0.772 smp/s:38 mem_gb:39.59
60
+ Training: 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 8000/10000 [3:41:01<58:47, 1.76s/step]INFO 2026-07-15 18:14:14 ot_train.py:646 step:58K smpl:4M ep:84K epch:17.00 loss:0.019 grdn:0.219 lr:3.1e-07 updt_s:0.915 data_s:0.766 smp/s:38 mem_gb:39.59
61
+ Training: 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 8200/10000 [3:46:37<50:06, 1.67s/step]INFO 2026-07-15 18:19:50 ot_train.py:646 step:58K smpl:4M ep:84K epch:17.06 loss:0.020 grdn:0.230 lr:2.5e-07 updt_s:0.910 data_s:0.768 smp/s:38 mem_gb:39.59
62
+ Training: 84%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 8400/10000 [3:52:19<43:53, 1.65s/step]INFO 2026-07-15 18:25:32 ot_train.py:646 step:58K smpl:4M ep:84K epch:17.12 loss:0.020 grdn:0.231 lr:2.0e-07 updt_s:0.936 data_s:0.770 smp/s:38 mem_gb:39.59
63
+ Training: 86%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 8600/10000 [3:57:55<42:26, 1.82s/step]INFO 2026-07-15 18:31:08 ot_train.py:646 step:59K smpl:4M ep:85K epch:17.17 loss:0.019 grdn:0.228 lr:1.6e-07 updt_s:0.953 data_s:0.725 smp/s:38 mem_gb:39.59
64
+ Training: 88%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 8800/10000 [4:03:31<33:19, 1.67s/step]INFO 2026-07-15 18:36:44 ot_train.py:646 step:59K smpl:4M ep:85K epch:17.23 loss:0.020 grdn:0.234 lr:1.2e-07 updt_s:0.902 data_s:0.778 smp/s:38 mem_gb:39.58
65
+ Training: 90%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 9000/10000 [4:09:07<30:05, 1.81s/step]INFO 2026-07-15 18:42:20 ot_train.py:646 step:59K smpl:4M ep:85K epch:17.29 loss:0.019 grdn:0.220 lr:8.4e-08 updt_s:0.904 data_s:0.771 smp/s:38 mem_gb:39.59
66
+ Training: 92%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 9200/10000 [4:14:44<23:27, 1.76s/step]INFO 2026-07-15 18:47:57 ot_train.py:646 step:59K smpl:4M ep:86K epch:17.35 loss:0.020 grdn:0.228 lr:5.7e-08 updt_s:0.899 data_s:0.782 smp/s:38 mem_gb:39.59
67
+ Training: 94%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 9400/10000 [4:20:17<16:53, 1.69s/step]INFO 2026-07-15 18:53:30 ot_train.py:646 step:59K smpl:4M ep:86K epch:17.41 loss:0.020 grdn:0.243 lr:3.4e-08 updt_s:0.901 data_s:0.764 smp/s:38 mem_gb:39.59
68
+ Training: 96%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ| 9600/10000 [4:25:50<10:42, 1.61s/step]INFO 2026-07-15 18:59:03 ot_train.py:646 step:60K smpl:4M ep:86K epch:17.47 loss:0.019 grdn:0.228 lr:1.8e-08 updt_s:0.909 data_s:0.752 smp/s:39 mem_gb:39.59
69
+ Training: 98%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š| 9800/10000 [4:31:24<05:26, 1.63s/step]INFO 2026-07-15 19:04:37 ot_train.py:646 step:60K smpl:4M ep:86K epch:17.53 loss:0.019 grdn:0.225 lr:6.5e-09 updt_s:0.889 data_s:0.778 smp/s:38 mem_gb:39.59
70
+ Training: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 10000/10000 [4:37:01<00:00, 1.66s/step]INFO 2026-07-15 19:10:14 ot_train.py:646 step:60K smpl:4M ep:87K epch:17.59 loss:0.019 grdn:0.219 lr:9.2e-10 updt_s:0.909 data_s:0.773 smp/s:38 mem_gb:39.59
71
+ INFO 2026-07-15 19:10:14 ot_train.py:691 Checkpoint policy after step 60000
72
+ INFO 2026-07-15 19:10:44 ain_yaml.py:918 Saved config, action-space, prompt, and phase provenance in /home/ext_minje/groot_insight/Abs_6D/Baseline and /home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/060000
73
+ Training: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 10000/10000 [4:37:31<00:00, 1.67s/step]
74
+ INFO 2026-07-15 19:10:44 ot_train.py:777 End of training
Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/requirements.txt ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pyarrow==25.0.0
2
+ importlib_metadata==9.0.0
3
+ wheel==0.47.0
4
+ cloudpickle==3.1.2
5
+ nvidia-cusolver-cu12==11.7.3.90
6
+ multiprocess==0.70.19
7
+ torch==2.11.0+cu128
8
+ nvidia-nvjitlink-cu12==12.8.93
9
+ fsspec==2026.2.0
10
+ pytz==2026.2
11
+ hf-xet==1.5.1
12
+ tokenizers==0.22.2
13
+ opencv-python-headless==4.13.0.92
14
+ cuda-pathfinder==1.2.2
15
+ frozenlist==1.8.0
16
+ smmap==5.0.3
17
+ httpcore==1.0.9
18
+ nvidia-nvshmem-cu12==3.4.5
19
+ nvidia-cuda-nvrtc-cu12==12.8.93
20
+ annotated-types==0.7.0
21
+ platformdirs==4.10.0
22
+ click==8.4.2
23
+ pyyaml-include==1.4.1
24
+ MarkupSafe==3.0.3
25
+ Jinja2==3.1.6
26
+ urllib3==2.7.0
27
+ nvidia-curand-cu12==10.3.9.90
28
+ certifi==2026.6.17
29
+ nvidia-cuda-runtime-cu12==12.8.90
30
+ torchcodec==0.11.1
31
+ toml==0.10.2
32
+ av==15.1.0
33
+ nvidia-cufft-cu12==11.3.3.83
34
+ attrs==26.1.0
35
+ lerobot==0.6.0
36
+ PyYAML==6.0.3
37
+ safetensors==0.8.0
38
+ diffusers==0.35.2
39
+ requests==2.34.2
40
+ nvidia-nvtx-cu12==12.8.90
41
+ charset-normalizer==3.4.9
42
+ networkx==3.6.1
43
+ cmake==4.1.3
44
+ pydantic_core==2.46.4
45
+ timm==1.0.28
46
+ dm-tree==0.1.10
47
+ nvidia-cufile-cu12==1.13.1.3
48
+ termcolor==3.3.0
49
+ multidict==6.7.1
50
+ filelock==3.29.0
51
+ tzdata==2026.3
52
+ aiohappyeyeballs==2.7.1
53
+ httpx==0.28.1
54
+ nvidia-nccl-cu12==2.28.9
55
+ Pygments==2.20.0
56
+ einops==0.8.2
57
+ nvidia-cusparse-cu12==12.5.8.93
58
+ torchvision==0.26.0+cu128
59
+ dill==0.4.1
60
+ GitPython==3.1.51
61
+ nvidia-cuda-cupti-cu12==12.8.90
62
+ decord==0.6.0
63
+ sympy==1.14.0
64
+ cuda-toolkit==12.8.1
65
+ python-dateutil==2.9.0.post0
66
+ shellingham==1.5.4
67
+ propcache==0.5.2
68
+ Farama-Notifications==0.0.6
69
+ pillow==12.2.0
70
+ cuda-bindings==12.9.4
71
+ huggingface_hub==1.23.0
72
+ pandas==2.3.3
73
+ typing-inspect==0.9.0
74
+ packaging==25.0
75
+ rich==15.0.0
76
+ nvidia-cusparselt-cu12==0.7.1
77
+ setuptools==78.1.0
78
+ wrapt==2.2.2
79
+ mypy_extensions==1.1.0
80
+ tqdm==4.68.4
81
+ draccus==0.10.0
82
+ aiosignal==1.4.0
83
+ mpmath==1.3.0
84
+ numpy==2.2.6
85
+ typing_extensions==4.15.0
86
+ psutil==7.2.2
87
+ mdurl==0.1.2
88
+ nvidia-cublas-cu12==12.8.4.1
89
+ absl-py==2.5.0
90
+ gitdb==4.0.12
91
+ transformers==5.5.4
92
+ accelerate==1.14.0
93
+ pydantic==2.13.4
94
+ pip==26.1.2
95
+ markdown-it-py==4.2.0
96
+ typer==0.26.8
97
+ triton==3.6.0
98
+ wandb==0.27.2
99
+ regex==2026.7.10
100
+ yarl==1.24.2
101
+ jsonlines==4.0.0
102
+ mergedeep==1.3.4
103
+ aiohttp==3.14.1
104
+ datasets==4.8.5
105
+ gymnasium==1.3.0
106
+ peft==0.19.1
107
+ zipp==4.1.0
108
+ idna==3.18
109
+ nvidia-cudnn-cu12==9.19.0.56
110
+ h11==0.16.0
111
+ sentry-sdk==2.65.0
112
+ annotated-doc==0.0.4
113
+ protobuf==7.35.1
114
+ typing-inspection==0.4.2
115
+ anyio==4.14.2
116
+ xxhash==3.8.1
117
+ six==1.17.0
Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/wandb-metadata.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39",
3
+ "python": "CPython 3.12.13",
4
+ "startedAt": "2026-07-15T05:32:50.279073Z",
5
+ "args": [
6
+ "--config_path=/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model/train_config.json",
7
+ "--resume=true"
8
+ ],
9
+ "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
10
+ "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
11
+ "codePathLocal": "my_scripts/lerobot_train_yaml.py",
12
+ "git": {
13
+ "remote": "git@github.com:paragon7060/clvla.git",
14
+ "commit": "22b2faefeaab966ca2dc3688fe6a77f3bcd8de32"
15
+ },
16
+ "root": "/home/ext_minje/groot_insight/Abs_6D/Baseline",
17
+ "host": "DGX-H200-01",
18
+ "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python",
19
+ "cpu_count": 112,
20
+ "cpu_count_logical": 224,
21
+ "gpu": "NVIDIA H200",
22
+ "gpu_count": 8,
23
+ "disk": {
24
+ "/": {
25
+ "total": "1888425144320",
26
+ "used": "1297062400000"
27
+ }
28
+ },
29
+ "memory": {
30
+ "total": "2164177248256"
31
+ },
32
+ "gpu_nvidia": [
33
+ {
34
+ "name": "NVIDIA H200",
35
+ "memoryTotal": "150754820096",
36
+ "cudaCores": 16896,
37
+ "architecture": "Hopper",
38
+ "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3"
39
+ },
40
+ {
41
+ "name": "NVIDIA H200",
42
+ "memoryTotal": "150754820096",
43
+ "cudaCores": 16896,
44
+ "architecture": "Hopper",
45
+ "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb"
46
+ },
47
+ {
48
+ "name": "NVIDIA H200",
49
+ "memoryTotal": "150754820096",
50
+ "cudaCores": 16896,
51
+ "architecture": "Hopper",
52
+ "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275"
53
+ },
54
+ {
55
+ "name": "NVIDIA H200",
56
+ "memoryTotal": "150754820096",
57
+ "cudaCores": 16896,
58
+ "architecture": "Hopper",
59
+ "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21"
60
+ },
61
+ {
62
+ "name": "NVIDIA H200",
63
+ "memoryTotal": "150754820096",
64
+ "cudaCores": 16896,
65
+ "architecture": "Hopper",
66
+ "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d"
67
+ },
68
+ {
69
+ "name": "NVIDIA H200",
70
+ "memoryTotal": "150754820096",
71
+ "cudaCores": 16896,
72
+ "architecture": "Hopper",
73
+ "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd"
74
+ },
75
+ {
76
+ "name": "NVIDIA H200",
77
+ "memoryTotal": "150754820096",
78
+ "cudaCores": 16896,
79
+ "architecture": "Hopper",
80
+ "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f"
81
+ },
82
+ {
83
+ "name": "NVIDIA H200",
84
+ "memoryTotal": "150754820096",
85
+ "cudaCores": 16896,
86
+ "architecture": "Hopper",
87
+ "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c"
88
+ }
89
+ ],
90
+ "cudaVersion": "12.9",
91
+ "writerId": "c8d5ybgsruh4m5h3ie5jd18btesa3xht"
92
+ }
Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_step":60000,"_wandb":{"runtime":97011},"_timestamp":1.7841102141629555e+09,"train/loss":0.020864611491560936,"train/update_s":0.909430973721901,"train/steps":60000,"train/lr":9.22312585679652e-10,"train/samples_per_s":38.035328348668635,"train/dataloading_s":0.773215207568137,"train/episodes":86694.41811262691,"train/grad_norm":0.2188143553212285,"train/epochs":17.58507466787564,"train/gpu_mem_gb":39.59298655033112,"train/samples":3840000,"_runtime":97011.728859976}
Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug.log ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
2
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_setup.py:_flush():81] Configure stats pid to 1450496
3
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug.log
5
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/Abs_6D/Baseline/wandb/run-20260715_143250-1jmjb68j/logs/debug-internal.log
6
+ 2026-07-15 14:32:50,279 INFO MainThread:1450496 [wandb_init.py:init():766] calling init triggers
7
+ 2026-07-15 14:32:50,280 INFO MainThread:1450496 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
8
+ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/Abs_6D/Baseline', 'job_name': 'INSIGHT_6D_baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '1jmjb68j', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000', '_wandb': {}}
9
+ 2026-07-15 14:32:50,280 INFO MainThread:1450496 [wandb_init.py:init():814] starting backend
10
+ 2026-07-15 14:32:50,495 INFO MainThread:1450496 [wandb_init.py:init():829] sending inform_init request
11
+ 2026-07-15 14:32:50,989 INFO MainThread:1450496 [wandb_init.py:init():834] backend started and connected
12
+ 2026-07-15 14:32:50,999 INFO MainThread:1450496 [wandb_init.py:init():904] updated telemetry
13
+ 2026-07-15 14:32:51,006 INFO MainThread:1450496 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
14
+ 2026-07-15 14:32:51,687 INFO MainThread:1450496 [wandb_init.py:init():967] run resumed
15
+ 2026-07-15 14:32:51,692 INFO MainThread:1450496 [wandb_init.py:init():972] starting run threads in backend
16
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_console_start():2523] atexit reg
17
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_redirect():2373] redirect: wrap_raw
18
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_redirect():2442] Wrapping output streams.
19
+ 2026-07-15 14:32:51,783 INFO MainThread:1450496 [wandb_run.py:_redirect():2465] Redirects installed.
20
+ 2026-07-15 14:32:51,785 INFO MainThread:1450496 [wandb_init.py:init():1010] run started, returning control to user process
21
+ 2026-07-15 19:10:44,422 INFO wandb-AsyncioManager-main:1450496 [service_client.py:_forward_responses():122] Reached EOF.
22
+ 2026-07-15 19:10:44,422 INFO wandb-AsyncioManager-main:1450496 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
23
+ 2026-07-15 19:10:44,423 ERROR wandb-AsyncioManager-main:1450496 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
24
+ Traceback (most recent call last):
25
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
26
+ await fn()
27
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
28
+ await asyncio_compat.race(
29
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
30
+ async with open_task_group(race=True) as tg:
31
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
32
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
33
+ await anext(self.gen)
34
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
35
+ await task_group._wait_all(race=race, timeout=exit_timeout)
36
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
37
+ raise exc
38
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
39
+ result = await handle.wait_async(timeout=None)
40
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
41
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
42
+ response = await self._handle.wait_async(timeout=timeout)
43
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
44
+ File "/home/ext_minje/miniforge3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
45
+ raise HandleAbandonedError()
46
+ wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
Abs_6D/Baseline_gpu3_resume.pid ADDED
@@ -0,0 +1 @@
 
 
1
+ 1450496
Abs_6D/Baseline_gpu3_watcher_status.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "error": "resumed process environment mismatch: {'INSIGHT_LEROBOT_ACTION_SPACE': {'actual': None, 'expected': 'ee_abs_rot6d'}, 'INSIGHT_LEROBOT_PROMPT_SET': {'actual': None, 'expected': 'detailed'}, 'INSIGHT_LEROBOT_SAVE_STEPS': {'actual': None, 'expected': '60000'}}",
3
+ "state": "error",
4
+ "updated_at": "2026-07-15T14:33:15+09:00"
5
+ }
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/phase_schedule.json ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "step_interval": "[start_step, until_step)",
4
+ "total_steps": 60000,
5
+ "optimizer_lifecycle": "single_continuous_optimizer",
6
+ "scheduler_lifecycle": "single_continuous_scheduler",
7
+ "phases": [
8
+ {
9
+ "index": 0,
10
+ "name": "lap_rkd_linear_decay_6K",
11
+ "start_step": 0,
12
+ "until_step": 6000,
13
+ "policy": {
14
+ "rkd_enabled": true,
15
+ "rkd_fm_loss_weight": 1.0,
16
+ "rkd_loss_weight": 1.0,
17
+ "rkd_loss_weight_end": 0.0,
18
+ "rkd_loss_weight_schedule": "linear",
19
+ "rkd_distance_weight": 1.0,
20
+ "rkd_angle_weight": 2.0
21
+ },
22
+ "trainable": {
23
+ "tune_llm": false,
24
+ "tune_visual": false,
25
+ "tune_top_llm_layers": 0,
26
+ "tune_projector": true,
27
+ "tune_diffusion_model": true,
28
+ "tune_vlln": true
29
+ }
30
+ },
31
+ {
32
+ "index": 1,
33
+ "name": "fm_only",
34
+ "start_step": 6000,
35
+ "until_step": 60000,
36
+ "policy": {
37
+ "rkd_enabled": false,
38
+ "rkd_fm_loss_weight": 1.0,
39
+ "rkd_loss_weight": 0.0,
40
+ "rkd_loss_weight_end": null,
41
+ "rkd_loss_weight_schedule": "linear",
42
+ "rkd_distance_weight": 1.0,
43
+ "rkd_angle_weight": 2.0
44
+ },
45
+ "trainable": {
46
+ "tune_llm": false,
47
+ "tune_visual": false,
48
+ "tune_top_llm_layers": 0,
49
+ "tune_projector": true,
50
+ "tune_diffusion_model": true,
51
+ "tune_vlln": true
52
+ }
53
+ }
54
+ ]
55
+ }
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/pretrained_model/action_space_manifest.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "action_space": "ee_abs_rot6d",
4
+ "source_feature": "action.ee_abs_rot6d",
5
+ "canonical_feature": "action",
6
+ "dtype": "float32",
7
+ "shape": [
8
+ 10
9
+ ],
10
+ "names": [
11
+ "x",
12
+ "y",
13
+ "z",
14
+ "r_col0_x",
15
+ "r_col0_y",
16
+ "r_col0_z",
17
+ "r_col1_x",
18
+ "r_col1_y",
19
+ "r_col1_z",
20
+ "gripper"
21
+ ],
22
+ "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
23
+ "dataset_revision": "v3.0",
24
+ "dataset_total_frames": 218367,
25
+ "dataset_total_episodes": 4930,
26
+ "dataset_fps": 10,
27
+ "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4",
28
+ "stats": {
29
+ "min": [
30
+ -0.21784618496894836,
31
+ -0.4474811851978302,
32
+ 0.1702737659215927,
33
+ -0.9999986290931702,
34
+ -0.9999991655349731,
35
+ -0.9999999403953552,
36
+ -0.9999990463256836,
37
+ -0.9999998211860657,
38
+ -0.9999988079071045,
39
+ 0.0
40
+ ],
41
+ "max": [
42
+ 0.6283698678016663,
43
+ 0.4420371949672699,
44
+ 1.051027774810791,
45
+ 0.9999999403953552,
46
+ 0.9999995231628418,
47
+ 1.0,
48
+ 0.9999995231628418,
49
+ 1.0,
50
+ 0.9999996423721313,
51
+ 0.03999999910593033
52
+ ],
53
+ "mean": [
54
+ 0.4040228037735655,
55
+ -0.05247618696989652,
56
+ 0.6993459771218912,
57
+ 0.07011111991957117,
58
+ -0.03520558904460197,
59
+ 0.14470242846953205,
60
+ 0.05491166606739672,
61
+ -0.19073058893378805,
62
+ -0.06988117661871228,
63
+ 0.021167300266524632
64
+ ],
65
+ "std": [
66
+ 0.11861927913357073,
67
+ 0.11318943557281885,
68
+ 0.17849160646987516,
69
+ 0.4812528791886511,
70
+ 0.7487196258736938,
71
+ 0.4251126804030219,
72
+ 0.47797229807769137,
73
+ 0.6267695346407992,
74
+ 0.5782954306815554,
75
+ 0.01996590431050817
76
+ ],
77
+ "count": [
78
+ 218367
79
+ ],
80
+ "q01": [
81
+ 0.23988881827972064,
82
+ -0.10387461883034811,
83
+ 0.6297243923476871,
84
+ -0.43622841955353464,
85
+ -0.6885486527617446,
86
+ -0.14103555388039019,
87
+ -0.4327210063987595,
88
+ -0.9304790915716145,
89
+ -0.3465462258490417,
90
+ -1.000000013351432e-10
91
+ ],
92
+ "q10": [
93
+ 0.3042306436755216,
94
+ -0.09609991620622624,
95
+ 0.6394063072781145,
96
+ -0.3351306352998669,
97
+ -0.5751963015002924,
98
+ -0.08490621283307866,
99
+ -0.33849701676224275,
100
+ -0.8340557712529717,
101
+ -0.29760290790884514,
102
+ -1.000000013351432e-10
103
+ ],
104
+ "q50": [
105
+ 0.4160225263528291,
106
+ -0.06267459152989589,
107
+ 0.6811777491060801,
108
+ 0.0543536902230377,
109
+ -0.14012251058819217,
110
+ 0.04772394280004727,
111
+ 0.03838119138716219,
112
+ -0.14865169094195635,
113
+ -0.2073069401172425,
114
+ 0.020772594934870346
115
+ ],
116
+ "q90": [
117
+ 0.46641008755679864,
118
+ 0.020233143101890064,
119
+ 0.8095135405259374,
120
+ 0.48172401592474906,
121
+ 0.7398737043070319,
122
+ 0.5970841215636613,
123
+ 0.4674752875999555,
124
+ 0.36281754777672676,
125
+ 0.5033361883834496,
126
+ 0.039998362213352394
127
+ ],
128
+ "q99": [
129
+ 0.47320723171849244,
130
+ 0.04114262476135705,
131
+ 0.8467967259270389,
132
+ 0.5867781409980988,
133
+ 0.8529584859257228,
134
+ 0.7953936054341789,
135
+ 0.5768269858732034,
136
+ 0.49353349323389695,
137
+ 0.7168914440776504,
138
+ 0.03999983541667277
139
+ ]
140
+ }
141
+ }
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/pretrained_model/policy_postprocessor.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "groot_action_unpack_unnormalize_v2",
6
+ "config": {
7
+ "env_action_dim": 10,
8
+ "normalize_min_max": true,
9
+ "clip_normalized_action": true,
10
+ "libero_gripper_action": false,
11
+ "libero_gripper_binarize": true
12
+ },
13
+ "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
14
+ },
15
+ {
16
+ "registry_name": "device_processor",
17
+ "config": {
18
+ "device": "cpu",
19
+ "float_dtype": null
20
+ }
21
+ }
22
+ ]
23
+ }
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/pretrained_model/train_config.json ADDED
@@ -0,0 +1,268 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
4
+ "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "tfs": {
11
+ "brightness": {
12
+ "weight": 1.0,
13
+ "type": "ColorJitter",
14
+ "kwargs": {
15
+ "brightness": [
16
+ 0.8,
17
+ 1.2
18
+ ]
19
+ }
20
+ },
21
+ "contrast": {
22
+ "weight": 1.0,
23
+ "type": "ColorJitter",
24
+ "kwargs": {
25
+ "contrast": [
26
+ 0.8,
27
+ 1.2
28
+ ]
29
+ }
30
+ },
31
+ "saturation": {
32
+ "weight": 1.0,
33
+ "type": "ColorJitter",
34
+ "kwargs": {
35
+ "saturation": [
36
+ 0.5,
37
+ 1.5
38
+ ]
39
+ }
40
+ },
41
+ "hue": {
42
+ "weight": 1.0,
43
+ "type": "ColorJitter",
44
+ "kwargs": {
45
+ "hue": [
46
+ -0.05,
47
+ 0.05
48
+ ]
49
+ }
50
+ },
51
+ "sharpness": {
52
+ "weight": 1.0,
53
+ "type": "SharpnessJitter",
54
+ "kwargs": {
55
+ "sharpness": [
56
+ 0.5,
57
+ 1.5
58
+ ]
59
+ }
60
+ },
61
+ "affine": {
62
+ "weight": 1.0,
63
+ "type": "RandomAffine",
64
+ "kwargs": {
65
+ "degrees": [
66
+ -5.0,
67
+ 5.0
68
+ ],
69
+ "translate": [
70
+ 0.05,
71
+ 0.05
72
+ ]
73
+ }
74
+ }
75
+ }
76
+ },
77
+ "revision": null,
78
+ "use_imagenet_stats": true,
79
+ "video_backend": "torchcodec",
80
+ "return_uint8": false,
81
+ "depth_output_unit": "mm",
82
+ "streaming": false,
83
+ "eval_split": 0.0
84
+ },
85
+ "env": null,
86
+ "policy": {
87
+ "type": "groot_rkd",
88
+ "n_obs_steps": 1,
89
+ "input_features": {
90
+ "observation.state": {
91
+ "type": "STATE",
92
+ "shape": [
93
+ 16
94
+ ]
95
+ },
96
+ "observation.images.wrist": {
97
+ "type": "VISUAL",
98
+ "shape": [
99
+ 3,
100
+ 224,
101
+ 224
102
+ ]
103
+ },
104
+ "observation.images.right_shoulder": {
105
+ "type": "VISUAL",
106
+ "shape": [
107
+ 3,
108
+ 224,
109
+ 224
110
+ ]
111
+ },
112
+ "observation.images.guide": {
113
+ "type": "VISUAL",
114
+ "shape": [
115
+ 3,
116
+ 224,
117
+ 224
118
+ ]
119
+ }
120
+ },
121
+ "output_features": {
122
+ "action": {
123
+ "type": "ACTION",
124
+ "shape": [
125
+ 10
126
+ ]
127
+ }
128
+ },
129
+ "device": "cuda",
130
+ "use_amp": false,
131
+ "use_peft": false,
132
+ "push_to_hub": false,
133
+ "repo_id": null,
134
+ "private": null,
135
+ "tags": null,
136
+ "license": null,
137
+ "pretrained_path": null,
138
+ "pretrained_revision": null,
139
+ "chunk_size": 16,
140
+ "n_action_steps": 16,
141
+ "max_state_dim": 132,
142
+ "max_action_dim": 132,
143
+ "normalization_mapping": {
144
+ "VISUAL": "IDENTITY",
145
+ "STATE": "IDENTITY",
146
+ "ACTION": "IDENTITY"
147
+ },
148
+ "base_model_path": "nvidia/GR00T-N1.7-3B",
149
+ "action_decode_transform": null,
150
+ "embodiment_tag": "new_embodiment",
151
+ "tune_llm": false,
152
+ "tune_visual": false,
153
+ "tune_projector": true,
154
+ "tune_diffusion_model": true,
155
+ "tune_vlln": true,
156
+ "tune_top_llm_layers": 0,
157
+ "num_inference_timesteps": null,
158
+ "rtc_ramp_rate": null,
159
+ "use_flash_attention": false,
160
+ "use_relative_actions": false,
161
+ "relative_exclude_joints": [],
162
+ "optimizer_lr": 0.0001,
163
+ "optimizer_betas": [
164
+ 0.9,
165
+ 0.999
166
+ ],
167
+ "optimizer_eps": 1e-08,
168
+ "optimizer_weight_decay": 1e-05,
169
+ "warmup_steps": 6000,
170
+ "warmup_ratio": 0.05,
171
+ "use_bf16": true,
172
+ "model_params_fp32": true,
173
+ "image_size": [
174
+ 256,
175
+ 256
176
+ ],
177
+ "tokenizer_assets_repo": null,
178
+ "lora_rank": 0,
179
+ "lora_alpha": 16,
180
+ "lora_dropout": 0.1,
181
+ "lora_full_model": false,
182
+ "video_backend": "decord",
183
+ "balance_dataset_weights": true,
184
+ "balance_trajectory_weights": true,
185
+ "dataset_paths": null,
186
+ "output_dir": "./tmp/gr00t",
187
+ "save_steps": 1000,
188
+ "max_steps": 10000,
189
+ "batch_size": 32,
190
+ "dataloader_num_workers": 8,
191
+ "report_to": "wandb",
192
+ "resume": false,
193
+ "rkd_enabled": true,
194
+ "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11",
195
+ "rkd_student_pooling": "attention",
196
+ "rkd_student_attention_hidden_dim": 512,
197
+ "rkd_student_attention_output_dim": 512,
198
+ "rkd_fm_loss_weight": 1.0,
199
+ "rkd_loss_weight": 1.0,
200
+ "rkd_distance_weight": 1.0,
201
+ "rkd_angle_weight": 2.0
202
+ },
203
+ "reward_model": null,
204
+ "output_dir": "/home/ext_minje/groot_insight/Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K",
205
+ "job_name": "INSIGHT_6D_RKD_TimewarpVAE_LAPstyle_6K",
206
+ "resume": false,
207
+ "seed": 42,
208
+ "cudnn_deterministic": false,
209
+ "num_workers": 4,
210
+ "batch_size": 64,
211
+ "prefetch_factor": 4,
212
+ "persistent_workers": true,
213
+ "steps": 60000,
214
+ "env_eval_freq": 20000,
215
+ "log_freq": 200,
216
+ "eval_steps": 0,
217
+ "max_eval_samples": 0,
218
+ "tolerance_s": 0.0001,
219
+ "save_checkpoint": true,
220
+ "save_freq": 20000,
221
+ "use_policy_training_preset": true,
222
+ "optimizer": {
223
+ "type": "adamw",
224
+ "lr": 0.0001,
225
+ "weight_decay": 1e-05,
226
+ "grad_clip_norm": 1.0,
227
+ "betas": [
228
+ 0.9,
229
+ 0.999
230
+ ],
231
+ "eps": 1e-08
232
+ },
233
+ "scheduler": {
234
+ "type": "diffuser",
235
+ "num_warmup_steps": 6000,
236
+ "name": "cosine"
237
+ },
238
+ "eval": {
239
+ "n_episodes": 50,
240
+ "batch_size": 50,
241
+ "use_async_envs": true,
242
+ "recording": false,
243
+ "recording_repo_id": null,
244
+ "recording_private": false
245
+ },
246
+ "wandb": {
247
+ "enable": true,
248
+ "disable_artifact": true,
249
+ "project": "lerobot",
250
+ "entity": null,
251
+ "notes": null,
252
+ "run_id": "pebh9ode",
253
+ "mode": null,
254
+ "add_tags": true
255
+ },
256
+ "peft": null,
257
+ "job": {
258
+ "target": null,
259
+ "image": "huggingface/lerobot-gpu:latest",
260
+ "timeout": "2d",
261
+ "detach": false,
262
+ "tags": []
263
+ },
264
+ "save_checkpoint_to_hub": false,
265
+ "sample_weighting": null,
266
+ "rename_map": {},
267
+ "checkpoint_path": null
268
+ }
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/resolved_config.yaml ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
3
+ root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2
4
+ policy:
5
+ type: groot_rkd
6
+ device: cuda
7
+ chunk_size: 16
8
+ n_action_steps: 16
9
+ push_to_hub: false
10
+ warmup_steps: 6000
11
+ tune_llm: false
12
+ tune_visual: false
13
+ tune_top_llm_layers: 0
14
+ tune_projector: true
15
+ tune_diffusion_model: true
16
+ tune_vlln: true
17
+ rkd_enabled: true
18
+ rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11
19
+ rkd_student_pooling: attention
20
+ rkd_student_attention_hidden_dim: 512
21
+ rkd_student_attention_output_dim: 512
22
+ rkd_fm_loss_weight: 1.0
23
+ rkd_loss_weight: 1.0
24
+ rkd_distance_weight: 1.0
25
+ rkd_angle_weight: 2.0
26
+ seed: 42
27
+ batch_size: 64
28
+ steps: 60000
29
+ log_freq: 200
30
+ output_dir: /home/ext_minje/groot_insight/Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K
31
+ job_name: INSIGHT_6D_RKD_TimewarpVAE_LAPstyle_6K
32
+ wandb:
33
+ enable: true
34
+ disable_artifact: true
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000/training_state/training_step.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "step": 60000,
3
+ "num_processes": 1,
4
+ "batch_size": 64
5
+ }
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/files/output.log ADDED
@@ -0,0 +1,321 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO 2026-07-15 01:14:49 db_utils.py:121 Logs will be synced with wandb.
2
+ INFO 2026-07-15 01:14:49 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/pebh9ode
3
+ INFO 2026-07-15 01:14:49 ot_train.py:298 Creating dataset
4
+ INFO 2026-07-15 01:14:51 ot_train.py:332 Creating policy
5
+ Fetching 27 files: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 27/27 [00:00<00:00, 1325.37it/s]
6
+ `torch_dtype` is deprecated! Use `dtype` instead!
7
+ Loading weights: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1031/1031 [00:00<00:00, 2096.45it/s]
8
+ INFO 2026-07-15 01:15:00 ot_train.py:405 Creating optimizer and scheduler
9
+ INFO 2026-07-15 01:15:00 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K
10
+ INFO 2026-07-15 01:15:00 ot_train.py:446 cfg.steps=60000 (60K)
11
+ INFO 2026-07-15 01:15:00 ot_train.py:447 dataset.num_frames=218367 (218K)
12
+ INFO 2026-07-15 01:15:00 ot_train.py:448 dataset.num_episodes=4930
13
+ INFO 2026-07-15 01:15:00 ot_train.py:451 Effective batch size: 64 x 1 = 64
14
+ INFO 2026-07-15 01:15:00 ot_train.py:452 num_learnable_params=1622090881 (2B)
15
+ INFO 2026-07-15 01:15:00 ot_train.py:453 num_total_params=3145709729 (3B)
16
+ Training: 0%| | 0/60000 [00:00<?, ?step/s]INFO 2026-07-15 01:15:00 ot_train.py:604 Start offline training on a fixed dataset, with effective batch size: 64
17
+ Training: 0%| | 200/60000 [04:46<22:37:09, 1.36s/step]INFO 2026-07-15 01:19:46 ot_train.py:649 step:200 smpl:13K ep:289 epch:0.06 loss:1.345 grdn:0.758 lr:1.7e-06 updt_s:0.833 data_s:0.594 smp/s:45 mem_gb:39.69
18
+ Training: 1%| | 400/60000 [09:24<22:24:07, 1.35s/step]INFO 2026-07-15 01:24:24 ot_train.py:649 step:400 smpl:26K ep:578 epch:0.12 loss:1.021 grdn:1.398 lr:5.0e-06 updt_s:0.813 data_s:0.574 smp/s:46 mem_gb:39.70
19
+ Training: 1%| | 600/60000 [14:03<21:52:56, 1.33s/step]INFO 2026-07-15 01:29:03 ot_train.py:649 step:600 smpl:38K ep:867 epch:0.18 loss:0.416 grdn:2.294 lr:8.3e-06 updt_s:0.823 data_s:0.567 smp/s:46 mem_gb:39.70
20
+ Training: 1%|▏ | 800/60000 [18:40<22:47:14, 1.39s/step]INFO 2026-07-15 01:33:40 ot_train.py:649 step:800 smpl:51K ep:1K epch:0.23 loss:0.281 grdn:2.509 lr:1.2e-05 updt_s:0.820 data_s:0.560 smp/s:46 mem_gb:39.70
21
+ Training: 2%|▏ | 1000/60000 [23:17<23:07:02, 1.41s/step]INFO 2026-07-15 01:38:17 ot_train.py:649 step:1K smpl:64K ep:1K epch:0.29 loss:0.236 grdn:2.537 lr:1.5e-05 updt_s:0.819 data_s:0.562 smp/s:46 mem_gb:39.70
22
+ Training: 2%|▏ | 1200/60000 [27:52<21:59:48, 1.35s/step]INFO 2026-07-15 01:42:53 ot_train.py:649 step:1K smpl:77K ep:2K epch:0.35 loss:0.203 grdn:2.252 lr:1.8e-05 updt_s:0.819 data_s:0.553 smp/s:47 mem_gb:39.70
23
+ Training: 2%|▏ | 1400/60000 [32:31<23:02:02, 1.42s/step]INFO 2026-07-15 01:47:31 ot_train.py:649 step:1K smpl:90K ep:2K epch:0.41 loss:0.186 grdn:1.998 lr:2.2e-05 updt_s:0.830 data_s:0.559 smp/s:46 mem_gb:39.70
24
+ Training: 3%|β–Ž | 1600/60000 [37:09<23:48:13, 1.47s/step]INFO 2026-07-15 01:52:09 ot_train.py:649 step:2K smpl:102K ep:2K epch:0.47 loss:0.173 grdn:1.758 lr:2.5e-05 updt_s:0.827 data_s:0.560 smp/s:46 mem_gb:39.70
25
+ Training: 3%|β–Ž | 1800/60000 [41:54<22:41:14, 1.40s/step]INFO 2026-07-15 01:56:54 ot_train.py:649 step:2K smpl:115K ep:3K epch:0.53 loss:0.162 grdn:1.516 lr:2.8e-05 updt_s:0.845 data_s:0.576 smp/s:45 mem_gb:39.70
26
+ Training: 3%|β–Ž | 2000/60000 [47:10<24:14:16, 1.50s/step]INFO 2026-07-15 02:02:10 ot_train.py:649 step:2K smpl:128K ep:3K epch:0.59 loss:0.153 grdn:1.335 lr:3.2e-05 updt_s:0.888 data_s:0.687 smp/s:41 mem_gb:39.70
27
+ Training: 4%|β–Ž | 2200/60000 [52:28<25:51:23, 1.61s/step]INFO 2026-07-15 02:07:29 ot_train.py:649 step:2K smpl:141K ep:3K epch:0.64 loss:0.147 grdn:1.223 lr:3.5e-05 updt_s:0.876 data_s:0.712 smp/s:40 mem_gb:39.70
28
+ Training: 4%|▍ | 2400/60000 [58:13<26:13:20, 1.64s/step]INFO 2026-07-15 02:13:13 ot_train.py:649 step:2K smpl:154K ep:3K epch:0.70 loss:0.142 grdn:1.103 lr:3.8e-05 updt_s:0.979 data_s:0.739 smp/s:37 mem_gb:39.69
29
+ Training: 4%|▍ | 2600/60000 [1:03:48<27:05:30, 1.70s/step]INFO 2026-07-15 02:18:48 ot_train.py:649 step:3K smpl:166K ep:4K epch:0.76 loss:0.137 grdn:0.987 lr:4.2e-05 updt_s:0.934 data_s:0.737 smp/s:38 mem_gb:39.71
30
+ Training: 5%|▍ | 2800/60000 [1:09:24<25:28:19, 1.60s/step]INFO 2026-07-15 02:24:25 ot_train.py:649 step:3K smpl:179K ep:4K epch:0.82 loss:0.132 grdn:0.912 lr:4.5e-05 updt_s:0.924 data_s:0.754 smp/s:38 mem_gb:39.71
31
+ Training: 5%|β–Œ | 3000/60000 [1:15:00<26:15:36, 1.66s/step]INFO 2026-07-15 02:30:01 ot_train.py:649 step:3K smpl:192K ep:4K epch:0.88 loss:0.125 grdn:0.805 lr:4.8e-05 updt_s:0.899 data_s:0.775 smp/s:38 mem_gb:39.71
32
+ Training: 5%|β–Œ | 3200/60000 [1:20:37<26:17:44, 1.67s/step]INFO 2026-07-15 02:35:37 ot_train.py:649 step:3K smpl:205K ep:5K epch:0.94 loss:0.123 grdn:0.761 lr:5.2e-05 updt_s:0.905 data_s:0.774 smp/s:38 mem_gb:39.71
33
+ Training: 6%|β–Œ | 3400/60000 [1:26:11<26:45:18, 1.70s/step]INFO 2026-07-15 02:41:12 ot_train.py:649 step:3K smpl:218K ep:5K epch:1.00 loss:0.117 grdn:0.685 lr:5.5e-05 updt_s:0.905 data_s:0.762 smp/s:38 mem_gb:39.71
34
+ Training: 6%|β–Œ | 3600/60000 [1:31:55<26:20:28, 1.68s/step]INFO 2026-07-15 02:46:55 ot_train.py:649 step:4K smpl:230K ep:5K epch:1.06 loss:0.113 grdn:0.646 lr:5.8e-05 updt_s:0.946 data_s:0.766 smp/s:37 mem_gb:39.71
35
+ Training: 6%|β–‹ | 3800/60000 [1:37:31<25:29:57, 1.63s/step]INFO 2026-07-15 02:52:31 ot_train.py:649 step:4K smpl:243K ep:5K epch:1.11 loss:0.110 grdn:0.636 lr:6.2e-05 updt_s:0.903 data_s:0.773 smp/s:38 mem_gb:39.71
36
+ Training: 7%|β–‹ | 4000/60000 [1:43:11<25:54:37, 1.67s/step]INFO 2026-07-15 02:58:11 ot_train.py:649 step:4K smpl:256K ep:6K epch:1.17 loss:0.109 grdn:0.590 lr:6.5e-05 updt_s:0.918 data_s:0.777 smp/s:38 mem_gb:39.71
37
+ Training: 7%|β–‹ | 4200/60000 [1:48:54<26:11:55, 1.69s/step]INFO 2026-07-15 03:03:54 ot_train.py:649 step:4K smpl:269K ep:6K epch:1.23 loss:0.107 grdn:0.556 lr:6.8e-05 updt_s:0.924 data_s:0.785 smp/s:37 mem_gb:39.71
38
+ Training: 7%|β–‹ | 4400/60000 [1:54:30<26:10:52, 1.70s/step]INFO 2026-07-15 03:09:31 ot_train.py:649 step:4K smpl:282K ep:6K epch:1.29 loss:0.105 grdn:0.524 lr:7.2e-05 updt_s:0.892 data_s:0.786 smp/s:38 mem_gb:39.71
39
+ Training: 8%|β–Š | 4600/60000 [2:00:05<25:36:21, 1.66s/step]INFO 2026-07-15 03:15:06 ot_train.py:649 step:5K smpl:294K ep:7K epch:1.35 loss:0.102 grdn:0.504 lr:7.5e-05 updt_s:0.878 data_s:0.791 smp/s:38 mem_gb:39.70
40
+ Training: 8%|β–Š | 4800/60000 [2:05:40<24:32:38, 1.60s/step]INFO 2026-07-15 03:20:41 ot_train.py:649 step:5K smpl:307K ep:7K epch:1.41 loss:0.102 grdn:0.516 lr:7.8e-05 updt_s:0.906 data_s:0.765 smp/s:38 mem_gb:39.71
41
+ Training: 8%|β–Š | 5000/60000 [2:11:18<26:55:04, 1.76s/step]INFO 2026-07-15 03:26:18 ot_train.py:649 step:5K smpl:320K ep:7K epch:1.47 loss:0.101 grdn:0.469 lr:8.2e-05 updt_s:0.899 data_s:0.783 smp/s:38 mem_gb:39.71
42
+ Training: 9%|β–Š | 5200/60000 [2:16:54<24:51:25, 1.63s/step]INFO 2026-07-15 03:31:55 ot_train.py:649 step:5K smpl:333K ep:8K epch:1.52 loss:0.103 grdn:0.472 lr:8.5e-05 updt_s:0.886 data_s:0.793 smp/s:38 mem_gb:39.71
43
+ Training: 9%|β–‰ | 5400/60000 [2:22:32<25:31:34, 1.68s/step]INFO 2026-07-15 03:37:33 ot_train.py:649 step:5K smpl:346K ep:8K epch:1.58 loss:0.098 grdn:0.447 lr:8.8e-05 updt_s:0.899 data_s:0.786 smp/s:38 mem_gb:39.71
44
+ Training: 9%|β–‰ | 5600/60000 [2:28:08<25:11:56, 1.67s/step]INFO 2026-07-15 03:43:08 ot_train.py:649 step:6K smpl:358K ep:8K epch:1.64 loss:0.096 grdn:0.432 lr:9.2e-05 updt_s:0.886 data_s:0.789 smp/s:38 mem_gb:39.71
45
+ Training: 10%|β–‰ | 5800/60000 [2:33:44<25:01:13, 1.66s/step]INFO 2026-07-15 03:48:44 ot_train.py:649 step:6K smpl:371K ep:8K epch:1.70 loss:0.094 grdn:0.417 lr:9.5e-05 updt_s:0.888 data_s:0.786 smp/s:38 mem_gb:39.71
46
+ Training: 10%|β–ˆ | 6000/60000 [2:39:21<27:26:41, 1.83s/step]INFO 2026-07-15 03:54:21 ot_train.py:649 step:6K smpl:384K ep:9K epch:1.76 loss:0.095 grdn:0.425 lr:9.8e-05 updt_s:0.889 data_s:0.789 smp/s:38 mem_gb:39.71
47
+ INFO 2026-07-15 03:54:21 ot_train.py:103 INSIGHT phase transition: index=1 name=fm_only range=[6000,60000) trainable_params=1620515968
48
+ Training: 10%|β–ˆ | 6200/60000 [2:44:57<25:12:11, 1.69s/step]INFO 2026-07-15 03:59:57 ot_train.py:649 step:6K smpl:397K ep:9K epch:1.82 loss:0.095 grdn:0.411 lr:1.0e-04 updt_s:0.898 data_s:0.778 smp/s:38 mem_gb:39.65
49
+ Training: 11%|β–ˆ | 6400/60000 [2:50:33<24:49:56, 1.67s/step]INFO 2026-07-15 04:05:33 ot_train.py:649 step:6K smpl:410K ep:9K epch:1.88 loss:0.092 grdn:0.406 lr:1.0e-04 updt_s:0.888 data_s:0.787 smp/s:38 mem_gb:39.65
50
+ Training: 11%|β–ˆ | 6600/60000 [2:56:11<24:40:30, 1.66s/step]INFO 2026-07-15 04:11:11 ot_train.py:649 step:7K smpl:422K ep:10K epch:1.93 loss:0.089 grdn:0.369 lr:1.0e-04 updt_s:0.899 data_s:0.785 smp/s:38 mem_gb:39.65
51
+ Training: 11%|β–ˆβ– | 6800/60000 [3:01:48<25:13:44, 1.71s/step]INFO 2026-07-15 04:16:49 ot_train.py:649 step:7K smpl:435K ep:10K epch:1.99 loss:0.092 grdn:0.408 lr:1.0e-04 updt_s:0.889 data_s:0.794 smp/s:38 mem_gb:39.63
52
+ Training: 12%|β–ˆβ– | 7000/60000 [3:07:23<24:04:04, 1.63s/step]INFO 2026-07-15 04:22:23 ot_train.py:649 step:7K smpl:448K ep:10K epch:2.05 loss:0.091 grdn:0.374 lr:1.0e-04 updt_s:0.889 data_s:0.779 smp/s:38 mem_gb:39.65
53
+ Training: 12%|β–ˆβ– | 7200/60000 [3:12:59<24:30:07, 1.67s/step]INFO 2026-07-15 04:27:59 ot_train.py:649 step:7K smpl:461K ep:10K epch:2.11 loss:0.087 grdn:0.365 lr:1.0e-04 updt_s:0.881 data_s:0.793 smp/s:38 mem_gb:39.65
54
+ Training: 12%|β–ˆβ– | 7400/60000 [3:18:36<24:22:07, 1.67s/step]INFO 2026-07-15 04:33:36 ot_train.py:649 step:7K smpl:474K ep:11K epch:2.17 loss:0.089 grdn:0.379 lr:1.0e-04 updt_s:0.888 data_s:0.791 smp/s:38 mem_gb:39.65
55
+ Training: 13%|β–ˆβ–Ž | 7600/60000 [3:24:13<24:14:16, 1.67s/step]INFO 2026-07-15 04:39:13 ot_train.py:649 step:8K smpl:486K ep:11K epch:2.23 loss:0.088 grdn:0.358 lr:1.0e-04 updt_s:0.891 data_s:0.791 smp/s:38 mem_gb:39.65
56
+ Training: 13%|β–ˆβ–Ž | 7800/60000 [3:29:51<27:32:12, 1.90s/step]INFO 2026-07-15 04:44:51 ot_train.py:649 step:8K smpl:499K ep:11K epch:2.29 loss:0.087 grdn:0.362 lr:1.0e-04 updt_s:0.901 data_s:0.782 smp/s:38 mem_gb:39.65
57
+ Training: 13%|β–ˆβ–Ž | 8000/60000 [3:35:26<24:11:18, 1.67s/step]INFO 2026-07-15 04:50:27 ot_train.py:649 step:8K smpl:512K ep:12K epch:2.34 loss:0.089 grdn:0.385 lr:1.0e-04 updt_s:0.870 data_s:0.803 smp/s:38 mem_gb:39.65
58
+ Training: 14%|β–ˆβ–Ž | 8200/60000 [3:41:06<23:27:19, 1.63s/step]INFO 2026-07-15 04:56:07 ot_train.py:649 step:8K smpl:525K ep:12K epch:2.40 loss:0.086 grdn:0.358 lr:1.0e-04 updt_s:0.909 data_s:0.787 smp/s:38 mem_gb:39.65
59
+ Training: 14%|β–ˆβ– | 8400/60000 [3:46:45<24:59:29, 1.74s/step]INFO 2026-07-15 05:01:45 ot_train.py:649 step:8K smpl:538K ep:12K epch:2.46 loss:0.085 grdn:0.354 lr:1.0e-04 updt_s:0.921 data_s:0.767 smp/s:38 mem_gb:39.65
60
+ Training: 14%|β–ˆβ– | 8600/60000 [3:52:23<24:03:08, 1.68s/step]INFO 2026-07-15 05:07:24 ot_train.py:649 step:9K smpl:550K ep:12K epch:2.52 loss:0.083 grdn:0.343 lr:9.9e-05 updt_s:0.906 data_s:0.781 smp/s:38 mem_gb:39.65
61
+ Training: 15%|β–ˆβ– | 8800/60000 [3:57:58<23:17:35, 1.64s/step]INFO 2026-07-15 05:12:59 ot_train.py:649 step:9K smpl:563K ep:13K epch:2.58 loss:0.084 grdn:0.336 lr:9.9e-05 updt_s:0.905 data_s:0.766 smp/s:38 mem_gb:39.64
62
+ Training: 15%|β–ˆβ–Œ | 9000/60000 [4:03:31<21:07:31, 1.49s/step]INFO 2026-07-15 05:18:32 ot_train.py:649 step:9K smpl:576K ep:13K epch:2.64 loss:0.081 grdn:0.331 lr:9.9e-05 updt_s:0.917 data_s:0.743 smp/s:39 mem_gb:39.65
63
+ Training: 15%|β–ˆβ–Œ | 9200/60000 [4:09:07<25:20:25, 1.80s/step]INFO 2026-07-15 05:24:07 ot_train.py:649 step:9K smpl:589K ep:13K epch:2.70 loss:0.084 grdn:0.379 lr:9.9e-05 updt_s:0.894 data_s:0.780 smp/s:38 mem_gb:39.63
64
+ Training: 16%|β–ˆβ–Œ | 9400/60000 [4:14:39<23:37:47, 1.68s/step]INFO 2026-07-15 05:29:39 ot_train.py:649 step:9K smpl:602K ep:14K epch:2.75 loss:0.086 grdn:0.377 lr:9.9e-05 updt_s:0.887 data_s:0.767 smp/s:39 mem_gb:39.65
65
+ Training: 16%|β–ˆβ–Œ | 9600/60000 [4:20:14<23:28:17, 1.68s/step]INFO 2026-07-15 05:35:15 ot_train.py:649 step:10K smpl:614K ep:14K epch:2.81 loss:0.082 grdn:0.334 lr:9.9e-05 updt_s:0.901 data_s:0.773 smp/s:38 mem_gb:39.65
66
+ Training: 16%|β–ˆβ–‹ | 9800/60000 [4:25:54<22:54:59, 1.64s/step]INFO 2026-07-15 05:40:55 ot_train.py:649 step:10K smpl:627K ep:14K epch:2.87 loss:0.080 grdn:0.326 lr:9.9e-05 updt_s:0.914 data_s:0.780 smp/s:38 mem_gb:39.65
67
+ Training: 17%|β–ˆβ–‹ | 10000/60000 [4:31:31<23:00:39, 1.66s/step]INFO 2026-07-15 05:46:31 ot_train.py:649 step:10K smpl:640K ep:14K epch:2.93 loss:0.077 grdn:0.322 lr:9.9e-05 updt_s:0.909 data_s:0.770 smp/s:38 mem_gb:39.65
68
+ Training: 17%|β–ˆβ–‹ | 10200/60000 [4:37:08<24:14:57, 1.75s/step]INFO 2026-07-15 05:52:08 ot_train.py:649 step:10K smpl:653K ep:15K epch:2.99 loss:0.078 grdn:0.321 lr:9.9e-05 updt_s:0.916 data_s:0.763 smp/s:38 mem_gb:39.65
69
+ Training: 17%|β–ˆβ–‹ | 10400/60000 [4:42:38<22:46:01, 1.65s/step]INFO 2026-07-15 05:57:38 ot_train.py:649 step:10K smpl:666K ep:15K epch:3.05 loss:0.078 grdn:0.321 lr:9.8e-05 updt_s:0.944 data_s:0.701 smp/s:39 mem_gb:39.65
70
+ Training: 18%|β–ˆβ–Š | 10600/60000 [4:48:10<23:12:41, 1.69s/step]INFO 2026-07-15 06:03:10 ot_train.py:649 step:11K smpl:678K ep:15K epch:3.11 loss:0.078 grdn:0.328 lr:9.8e-05 updt_s:0.915 data_s:0.739 smp/s:39 mem_gb:39.65
71
+ Training: 18%|β–ˆβ–Š | 10800/60000 [4:53:42<22:32:03, 1.65s/step]INFO 2026-07-15 06:08:43 ot_train.py:649 step:11K smpl:691K ep:16K epch:3.17 loss:0.080 grdn:0.335 lr:9.8e-05 updt_s:0.888 data_s:0.771 smp/s:39 mem_gb:39.65
72
+ Training: 18%|β–ˆβ–Š | 11000/60000 [4:59:14<22:51:50, 1.68s/step]INFO 2026-07-15 06:14:15 ot_train.py:649 step:11K smpl:704K ep:16K epch:3.22 loss:0.076 grdn:0.337 lr:9.8e-05 updt_s:0.892 data_s:0.764 smp/s:39 mem_gb:39.65
73
+ Training: 19%|β–ˆβ–Š | 11200/60000 [5:04:53<22:29:22, 1.66s/step]INFO 2026-07-15 06:19:53 ot_train.py:649 step:11K smpl:717K ep:16K epch:3.28 loss:0.076 grdn:0.318 lr:9.8e-05 updt_s:0.913 data_s:0.775 smp/s:38 mem_gb:39.65
74
+ Training: 19%|β–ˆβ–‰ | 11400/60000 [5:10:28<22:43:38, 1.68s/step]INFO 2026-07-15 06:25:29 ot_train.py:649 step:11K smpl:730K ep:16K epch:3.34 loss:0.078 grdn:0.324 lr:9.8e-05 updt_s:0.888 data_s:0.782 smp/s:38 mem_gb:39.63
75
+ Training: 19%|β–ˆβ–‰ | 11600/60000 [5:15:58<21:21:23, 1.59s/step]INFO 2026-07-15 06:30:59 ot_train.py:649 step:12K smpl:742K ep:17K epch:3.40 loss:0.075 grdn:0.322 lr:9.7e-05 updt_s:0.947 data_s:0.700 smp/s:39 mem_gb:39.65
76
+ Training: 20%|β–ˆβ–‰ | 11800/60000 [5:21:14<21:16:14, 1.59s/step]INFO 2026-07-15 06:36:15 ot_train.py:649 step:12K smpl:755K ep:17K epch:3.46 loss:0.077 grdn:0.333 lr:9.7e-05 updt_s:0.986 data_s:0.588 smp/s:41 mem_gb:39.65
77
+ Training: 20%|β–ˆβ–ˆ | 12000/60000 [5:26:27<21:17:46, 1.60s/step]INFO 2026-07-15 06:41:28 ot_train.py:649 step:12K smpl:768K ep:17K epch:3.52 loss:0.074 grdn:0.329 lr:9.7e-05 updt_s:0.982 data_s:0.580 smp/s:41 mem_gb:39.65
78
+ Training: 20%|β–ˆβ–ˆ | 12200/60000 [5:31:39<19:50:46, 1.49s/step]INFO 2026-07-15 06:46:39 ot_train.py:649 step:12K smpl:781K ep:18K epch:3.58 loss:0.075 grdn:0.330 lr:9.7e-05 updt_s:0.974 data_s:0.578 smp/s:41 mem_gb:39.65
79
+ Training: 21%|β–ˆβ–ˆ | 12400/60000 [5:36:48<19:14:09, 1.45s/step]INFO 2026-07-15 06:51:49 ot_train.py:649 step:12K smpl:794K ep:18K epch:3.63 loss:0.074 grdn:0.313 lr:9.7e-05 updt_s:0.963 data_s:0.581 smp/s:41 mem_gb:39.65
80
+ Training: 21%|β–ˆβ–ˆ | 12600/60000 [5:41:57<18:45:17, 1.42s/step]INFO 2026-07-15 06:56:58 ot_train.py:649 step:13K smpl:806K ep:18K epch:3.69 loss:0.074 grdn:0.307 lr:9.6e-05 updt_s:0.954 data_s:0.585 smp/s:42 mem_gb:39.65
81
+ Training: 21%|β–ˆβ–ˆβ– | 12800/60000 [5:47:06<20:03:30, 1.53s/step]INFO 2026-07-15 07:02:07 ot_train.py:649 step:13K smpl:819K ep:18K epch:3.75 loss:0.074 grdn:0.307 lr:9.6e-05 updt_s:0.958 data_s:0.582 smp/s:42 mem_gb:39.65
82
+ Training: 22%|β–ˆβ–ˆβ– | 13000/60000 [5:52:17<21:29:51, 1.65s/step]INFO 2026-07-15 07:07:18 ot_train.py:649 step:13K smpl:832K ep:19K epch:3.81 loss:0.072 grdn:0.296 lr:9.6e-05 updt_s:0.972 data_s:0.578 smp/s:41 mem_gb:39.65
83
+ Training: 22%|β–ˆβ–ˆβ– | 13200/60000 [5:57:32<19:53:07, 1.53s/step]INFO 2026-07-15 07:12:33 ot_train.py:649 step:13K smpl:845K ep:19K epch:3.87 loss:0.071 grdn:0.295 lr:9.6e-05 updt_s:0.986 data_s:0.584 smp/s:41 mem_gb:39.65
84
+ Training: 22%|β–ˆβ–ˆβ– | 13400/60000 [6:02:45<19:34:18, 1.51s/step]INFO 2026-07-15 07:17:45 ot_train.py:649 step:13K smpl:858K ep:19K epch:3.93 loss:0.073 grdn:0.300 lr:9.6e-05 updt_s:0.976 data_s:0.583 smp/s:41 mem_gb:39.65
85
+ Training: 23%|β–ˆβ–ˆβ–Ž | 13600/60000 [6:07:57<20:34:15, 1.60s/step]INFO 2026-07-15 07:22:57 ot_train.py:649 step:14K smpl:870K ep:20K epch:3.99 loss:0.072 grdn:0.314 lr:9.5e-05 updt_s:0.968 data_s:0.586 smp/s:41 mem_gb:39.63
86
+ Training: 23%|β–ˆβ–ˆβ–Ž | 13800/60000 [6:13:16<21:20:50, 1.66s/step]INFO 2026-07-15 07:28:16 ot_train.py:649 step:14K smpl:883K ep:20K epch:4.04 loss:0.071 grdn:0.299 lr:9.5e-05 updt_s:0.977 data_s:0.616 smp/s:40 mem_gb:39.65
87
+ Training: 23%|β–ˆβ–ˆβ–Ž | 14000/60000 [6:18:50<21:26:38, 1.68s/step]INFO 2026-07-15 07:33:50 ot_train.py:649 step:14K smpl:896K ep:20K epch:4.10 loss:0.070 grdn:0.306 lr:9.5e-05 updt_s:0.889 data_s:0.775 smp/s:38 mem_gb:39.65
88
+ Training: 24%|β–ˆβ–ˆβ–Ž | 14200/60000 [6:24:29<21:37:37, 1.70s/step]INFO 2026-07-15 07:39:30 ot_train.py:649 step:14K smpl:909K ep:21K epch:4.16 loss:0.070 grdn:0.299 lr:9.5e-05 updt_s:0.906 data_s:0.787 smp/s:38 mem_gb:39.65
89
+ Training: 24%|β–ˆβ–ˆβ– | 14400/60000 [6:30:02<21:12:34, 1.67s/step]INFO 2026-07-15 07:45:03 ot_train.py:649 step:14K smpl:922K ep:21K epch:4.22 loss:0.070 grdn:0.282 lr:9.4e-05 updt_s:0.891 data_s:0.769 smp/s:39 mem_gb:39.65
90
+ Training: 24%|β–ˆβ–ˆβ– | 14600/60000 [6:35:38<21:03:54, 1.67s/step]INFO 2026-07-15 07:50:39 ot_train.py:649 step:15K smpl:934K ep:21K epch:4.28 loss:0.070 grdn:0.308 lr:9.4e-05 updt_s:0.881 data_s:0.793 smp/s:38 mem_gb:39.65
91
+ Training: 25%|β–ˆβ–ˆβ– | 14800/60000 [6:41:15<20:02:20, 1.60s/step]INFO 2026-07-15 07:56:15 ot_train.py:649 step:15K smpl:947K ep:21K epch:4.34 loss:0.071 grdn:0.299 lr:9.4e-05 updt_s:0.888 data_s:0.788 smp/s:38 mem_gb:39.64
92
+ Training: 25%|β–ˆβ–ˆβ–Œ | 15000/60000 [6:46:50<21:03:54, 1.69s/step]INFO 2026-07-15 08:01:50 ot_train.py:649 step:15K smpl:960K ep:22K epch:4.40 loss:0.071 grdn:0.322 lr:9.3e-05 updt_s:0.892 data_s:0.780 smp/s:38 mem_gb:39.65
93
+ Training: 25%|β–ˆβ–ˆβ–Œ | 15200/60000 [6:52:28<21:02:19, 1.69s/step]INFO 2026-07-15 08:07:29 ot_train.py:649 step:15K smpl:973K ep:22K epch:4.45 loss:0.071 grdn:0.316 lr:9.3e-05 updt_s:0.902 data_s:0.785 smp/s:38 mem_gb:39.65
94
+ Training: 26%|β–ˆβ–ˆβ–Œ | 15400/60000 [6:58:02<20:36:48, 1.66s/step]INFO 2026-07-15 08:13:02 ot_train.py:649 step:15K smpl:986K ep:22K epch:4.51 loss:0.069 grdn:0.309 lr:9.3e-05 updt_s:0.879 data_s:0.783 smp/s:38 mem_gb:39.65
95
+ Training: 26%|β–ˆβ–ˆβ–Œ | 15600/60000 [7:03:37<20:47:22, 1.69s/step]INFO 2026-07-15 08:18:37 ot_train.py:649 step:16K smpl:998K ep:23K epch:4.57 loss:0.072 grdn:0.335 lr:9.3e-05 updt_s:0.881 data_s:0.791 smp/s:38 mem_gb:39.65
96
+ Training: 26%|β–ˆβ–ˆβ–‹ | 15800/60000 [7:09:12<26:23:47, 2.15s/step]INFO 2026-07-15 08:24:12 ot_train.py:649 step:16K smpl:1M ep:23K epch:4.63 loss:0.068 grdn:0.300 lr:9.2e-05 updt_s:0.871 data_s:0.798 smp/s:38 mem_gb:39.63
97
+ Training: 27%|β–ˆβ–ˆβ–‹ | 16000/60000 [7:14:46<20:01:54, 1.64s/step]INFO 2026-07-15 08:29:47 ot_train.py:649 step:16K smpl:1M ep:23K epch:4.69 loss:0.068 grdn:0.293 lr:9.2e-05 updt_s:0.873 data_s:0.794 smp/s:38 mem_gb:39.65
98
+ Training: 27%|β–ˆβ–ˆβ–‹ | 16200/60000 [7:20:24<20:17:45, 1.67s/step]INFO 2026-07-15 08:35:24 ot_train.py:649 step:16K smpl:1M ep:23K epch:4.75 loss:0.069 grdn:0.312 lr:9.2e-05 updt_s:0.883 data_s:0.800 smp/s:38 mem_gb:39.65
99
+ Training: 27%|β–ˆβ–ˆβ–‹ | 16400/60000 [7:26:03<20:11:57, 1.67s/step]INFO 2026-07-15 08:41:03 ot_train.py:649 step:16K smpl:1M ep:24K epch:4.81 loss:0.067 grdn:0.288 lr:9.1e-05 updt_s:0.899 data_s:0.790 smp/s:38 mem_gb:39.65
100
+ Training: 28%|β–ˆβ–ˆβ–Š | 16600/60000 [7:31:39<20:03:40, 1.66s/step]INFO 2026-07-15 08:46:40 ot_train.py:649 step:17K smpl:1M ep:24K epch:4.87 loss:0.065 grdn:0.283 lr:9.1e-05 updt_s:0.888 data_s:0.791 smp/s:38 mem_gb:39.65
101
+ Training: 28%|β–ˆβ–ˆβ–Š | 16800/60000 [7:37:16<19:40:27, 1.64s/step]INFO 2026-07-15 08:52:16 ot_train.py:649 step:17K smpl:1M ep:24K epch:4.92 loss:0.067 grdn:0.299 lr:9.1e-05 updt_s:0.888 data_s:0.789 smp/s:38 mem_gb:39.65
102
+ Training: 28%|β–ˆβ–ˆβ–Š | 17000/60000 [7:42:50<20:17:43, 1.70s/step]INFO 2026-07-15 08:57:51 ot_train.py:649 step:17K smpl:1M ep:25K epch:4.98 loss:0.065 grdn:0.289 lr:9.0e-05 updt_s:0.877 data_s:0.791 smp/s:38 mem_gb:39.65
103
+ Training: 29%|β–ˆβ–ˆβ–Š | 17200/60000 [7:48:25<20:13:04, 1.70s/step]INFO 2026-07-15 09:03:26 ot_train.py:649 step:17K smpl:1M ep:25K epch:5.04 loss:0.065 grdn:0.287 lr:9.0e-05 updt_s:0.886 data_s:0.784 smp/s:38 mem_gb:39.65
104
+ Training: 29%|β–ˆβ–ˆβ–‰ | 17400/60000 [7:53:57<18:59:10, 1.60s/step]INFO 2026-07-15 09:08:58 ot_train.py:649 step:17K smpl:1M ep:25K epch:5.10 loss:0.065 grdn:0.283 lr:9.0e-05 updt_s:0.878 data_s:0.778 smp/s:39 mem_gb:39.65
105
+ Training: 29%|β–ˆβ–ˆβ–‰ | 17600/60000 [7:59:33<20:48:12, 1.77s/step]INFO 2026-07-15 09:14:33 ot_train.py:649 step:18K smpl:1M ep:25K epch:5.16 loss:0.064 grdn:0.280 lr:8.9e-05 updt_s:0.893 data_s:0.778 smp/s:38 mem_gb:39.65
106
+ Training: 30%|β–ˆβ–ˆβ–‰ | 17800/60000 [8:05:05<19:23:52, 1.65s/step]INFO 2026-07-15 09:20:05 ot_train.py:649 step:18K smpl:1M ep:26K epch:5.22 loss:0.066 grdn:0.293 lr:8.9e-05 updt_s:0.863 data_s:0.793 smp/s:39 mem_gb:39.65
107
+ Training: 30%|β–ˆβ–ˆβ–ˆ | 18000/60000 [8:10:42<19:33:02, 1.68s/step]INFO 2026-07-15 09:25:42 ot_train.py:649 step:18K smpl:1M ep:26K epch:5.28 loss:0.064 grdn:0.276 lr:8.8e-05 updt_s:0.881 data_s:0.798 smp/s:38 mem_gb:39.65
108
+ Training: 30%|β–ˆβ–ˆβ–ˆ | 18200/60000 [8:16:17<19:24:20, 1.67s/step]INFO 2026-07-15 09:31:18 ot_train.py:649 step:18K smpl:1M ep:26K epch:5.33 loss:0.064 grdn:0.275 lr:8.8e-05 updt_s:0.874 data_s:0.799 smp/s:38 mem_gb:39.63
109
+ Training: 31%|β–ˆβ–ˆβ–ˆ | 18400/60000 [8:21:54<18:19:20, 1.59s/step]INFO 2026-07-15 09:36:54 ot_train.py:649 step:18K smpl:1M ep:27K epch:5.39 loss:0.062 grdn:0.273 lr:8.8e-05 updt_s:0.881 data_s:0.796 smp/s:38 mem_gb:39.65
110
+ Training: 31%|β–ˆβ–ˆβ–ˆ | 18600/60000 [8:27:30<19:21:28, 1.68s/step]INFO 2026-07-15 09:42:30 ot_train.py:649 step:19K smpl:1M ep:27K epch:5.45 loss:0.063 grdn:0.279 lr:8.7e-05 updt_s:0.879 data_s:0.795 smp/s:38 mem_gb:39.65
111
+ Training: 31%|β–ˆβ–ˆβ–ˆβ– | 18800/60000 [8:33:09<19:58:31, 1.75s/step]INFO 2026-07-15 09:48:09 ot_train.py:649 step:19K smpl:1M ep:27K epch:5.51 loss:0.065 grdn:0.321 lr:8.7e-05 updt_s:0.893 data_s:0.797 smp/s:38 mem_gb:39.65
112
+ Training: 32%|β–ˆβ–ˆβ–ˆβ– | 19000/60000 [8:37:57<16:03:07, 1.41s/step]INFO 2026-07-15 09:52:57 ot_train.py:649 step:19K smpl:1M ep:27K epch:5.57 loss:0.064 grdn:0.279 lr:8.7e-05 updt_s:0.814 data_s:0.621 smp/s:45 mem_gb:39.65
113
+ Training: 32%|β–ˆβ–ˆβ–ˆβ– | 19200/60000 [8:42:42<15:50:58, 1.40s/step]INFO 2026-07-15 09:57:42 ot_train.py:649 step:19K smpl:1M ep:28K epch:5.63 loss:0.064 grdn:0.282 lr:8.6e-05 updt_s:0.810 data_s:0.612 smp/s:45 mem_gb:39.65
114
+ Training: 32%|β–ˆβ–ˆβ–ˆβ– | 19400/60000 [8:47:13<15:03:45, 1.34s/step]INFO 2026-07-15 10:02:13 ot_train.py:649 step:19K smpl:1M ep:28K epch:5.69 loss:0.061 grdn:0.269 lr:8.6e-05 updt_s:0.801 data_s:0.549 smp/s:47 mem_gb:39.65
115
+ Training: 33%|β–ˆβ–ˆβ–ˆβ–Ž | 19600/60000 [8:51:41<14:50:00, 1.32s/step]INFO 2026-07-15 10:06:41 ot_train.py:649 step:20K smpl:1M ep:28K epch:5.74 loss:0.062 grdn:0.273 lr:8.5e-05 updt_s:0.803 data_s:0.534 smp/s:48 mem_gb:39.65
116
+ Training: 33%|β–ˆβ–ˆβ–ˆβ–Ž | 19800/60000 [8:56:08<14:47:32, 1.32s/step]INFO 2026-07-15 10:11:09 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.80 loss:0.061 grdn:0.279 lr:8.5e-05 updt_s:0.799 data_s:0.535 smp/s:48 mem_gb:39.65
117
+ Training: 33%|β–ˆβ–ˆβ–ˆβ–Ž | 20000/60000 [9:00:36<14:51:45, 1.34s/step]INFO 2026-07-15 10:15:37 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.86 loss:0.063 grdn:0.285 lr:8.5e-05 updt_s:0.802 data_s:0.535 smp/s:48 mem_gb:39.65
118
+ Training: 34%|β–ˆβ–ˆβ–ˆβ–Ž | 20200/60000 [9:05:07<14:42:18, 1.33s/step]INFO 2026-07-15 10:20:07 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.92 loss:0.061 grdn:0.271 lr:8.4e-05 updt_s:0.802 data_s:0.545 smp/s:48 mem_gb:39.65
119
+ Training: 34%|β–ˆβ–ˆβ–ˆβ– | 20400/60000 [9:09:39<14:47:23, 1.34s/step]INFO 2026-07-15 10:24:39 ot_train.py:649 step:20K smpl:1M ep:29K epch:5.98 loss:0.059 grdn:0.260 lr:8.4e-05 updt_s:0.813 data_s:0.545 smp/s:47 mem_gb:39.63
120
+ Training: 34%|β–ˆβ–ˆβ–ˆβ– | 20600/60000 [9:14:20<15:22:48, 1.41s/step]INFO 2026-07-15 10:29:21 ot_train.py:649 step:21K smpl:1M ep:30K epch:6.04 loss:0.060 grdn:0.267 lr:8.3e-05 updt_s:0.852 data_s:0.550 smp/s:46 mem_gb:39.65
121
+ Training: 35%|β–ˆβ–ˆβ–ˆβ– | 20800/60000 [9:18:51<14:42:18, 1.35s/step]INFO 2026-07-15 10:33:51 ot_train.py:649 step:21K smpl:1M ep:30K epch:6.10 loss:0.062 grdn:0.290 lr:8.3e-05 updt_s:0.809 data_s:0.539 smp/s:47 mem_gb:39.65
122
+ Training: 35%|β–ˆβ–ˆβ–ˆβ–Œ | 21000/60000 [9:23:20<14:31:23, 1.34s/step]INFO 2026-07-15 10:38:21 ot_train.py:649 step:21K smpl:1M ep:30K epch:6.15 loss:0.060 grdn:0.282 lr:8.2e-05 updt_s:0.807 data_s:0.537 smp/s:48 mem_gb:39.65
123
+ Training: 35%|β–ˆβ–ˆβ–ˆβ–Œ | 21200/60000 [9:27:50<14:25:41, 1.34s/step]INFO 2026-07-15 10:42:51 ot_train.py:649 step:21K smpl:1M ep:31K epch:6.21 loss:0.061 grdn:0.290 lr:8.2e-05 updt_s:0.810 data_s:0.537 smp/s:48 mem_gb:39.65
124
+ Training: 36%|β–ˆβ–ˆβ–ˆβ–Œ | 21400/60000 [9:32:20<14:22:03, 1.34s/step]INFO 2026-07-15 10:47:20 ot_train.py:649 step:21K smpl:1M ep:31K epch:6.27 loss:0.059 grdn:0.264 lr:8.1e-05 updt_s:0.801 data_s:0.543 smp/s:48 mem_gb:39.65
125
+ Training: 36%|β–ˆβ–ˆβ–ˆβ–Œ | 21600/60000 [9:36:50<14:07:18, 1.32s/step]INFO 2026-07-15 10:51:51 ot_train.py:649 step:22K smpl:1M ep:31K epch:6.33 loss:0.060 grdn:0.271 lr:8.1e-05 updt_s:0.801 data_s:0.547 smp/s:47 mem_gb:39.65
126
+ Training: 36%|β–ˆβ–ˆβ–ˆβ–‹ | 21800/60000 [9:41:20<13:51:06, 1.31s/step]INFO 2026-07-15 10:56:21 ot_train.py:649 step:22K smpl:1M ep:31K epch:6.39 loss:0.059 grdn:0.271 lr:8.1e-05 updt_s:0.807 data_s:0.539 smp/s:48 mem_gb:39.65
127
+ Training: 37%|β–ˆβ–ˆβ–ˆβ–‹ | 22000/60000 [9:46:06<15:11:06, 1.44s/step]INFO 2026-07-15 11:01:07 ot_train.py:649 step:22K smpl:1M ep:32K epch:6.45 loss:0.060 grdn:0.282 lr:8.0e-05 updt_s:0.816 data_s:0.608 smp/s:45 mem_gb:39.65
128
+ Training: 37%|β–ˆβ–ˆβ–ˆβ–‹ | 22200/60000 [9:50:44<14:18:25, 1.36s/step]INFO 2026-07-15 11:05:45 ot_train.py:649 step:22K smpl:1M ep:32K epch:6.51 loss:0.060 grdn:0.281 lr:8.0e-05 updt_s:0.810 data_s:0.576 smp/s:46 mem_gb:39.65
129
+ Training: 37%|β–ˆβ–ˆβ–ˆβ–‹ | 22400/60000 [9:55:14<14:02:04, 1.34s/step]INFO 2026-07-15 11:10:15 ot_train.py:649 step:22K smpl:1M ep:32K epch:6.57 loss:0.059 grdn:0.272 lr:7.9e-05 updt_s:0.804 data_s:0.543 smp/s:48 mem_gb:39.65
130
+ Training: 38%|β–ˆβ–ˆβ–ˆβ–Š | 22600/60000 [9:59:45<13:54:48, 1.34s/step]INFO 2026-07-15 11:14:45 ot_train.py:649 step:23K smpl:1M ep:33K epch:6.62 loss:0.058 grdn:0.282 lr:7.9e-05 updt_s:0.805 data_s:0.544 smp/s:47 mem_gb:39.63
131
+ Training: 38%|β–ˆβ–ˆβ–ˆβ–Š | 22800/60000 [10:04:14<14:10:21, 1.37s/step]INFO 2026-07-15 11:19:14 ot_train.py:649 step:23K smpl:1M ep:33K epch:6.68 loss:0.057 grdn:0.272 lr:7.8e-05 updt_s:0.798 data_s:0.542 smp/s:48 mem_gb:39.65
132
+ Training: 38%|β–ˆβ–ˆβ–ˆβ–Š | 23000/60000 [10:08:43<14:14:52, 1.39s/step]INFO 2026-07-15 11:23:43 ot_train.py:649 step:23K smpl:1M ep:33K epch:6.74 loss:0.056 grdn:0.270 lr:7.8e-05 updt_s:0.799 data_s:0.545 smp/s:48 mem_gb:39.65
133
+ Training: 39%|β–ˆβ–ˆβ–ˆβ–Š | 23200/60000 [10:13:14<13:53:02, 1.36s/step]INFO 2026-07-15 11:28:15 ot_train.py:649 step:23K smpl:1M ep:34K epch:6.80 loss:0.059 grdn:0.290 lr:7.7e-05 updt_s:0.809 data_s:0.542 smp/s:47 mem_gb:39.65
134
+ Training: 39%|β–ˆβ–ˆβ–ˆβ–‰ | 23400/60000 [10:17:44<13:48:05, 1.36s/step]INFO 2026-07-15 11:32:44 ot_train.py:649 step:23K smpl:1M ep:34K epch:6.86 loss:0.057 grdn:0.276 lr:7.7e-05 updt_s:0.802 data_s:0.543 smp/s:48 mem_gb:39.65
135
+ Training: 39%|β–ˆβ–ˆβ–ˆβ–‰ | 23600/60000 [10:22:14<13:30:40, 1.34s/step]INFO 2026-07-15 11:37:14 ot_train.py:649 step:24K smpl:2M ep:34K epch:6.92 loss:0.057 grdn:0.269 lr:7.6e-05 updt_s:0.807 data_s:0.538 smp/s:48 mem_gb:39.65
136
+ Training: 40%|β–ˆβ–ˆβ–ˆβ–‰ | 23800/60000 [10:26:53<14:29:06, 1.44s/step]INFO 2026-07-15 11:41:54 ot_train.py:649 step:24K smpl:2M ep:34K epch:6.98 loss:0.055 grdn:0.266 lr:7.6e-05 updt_s:0.822 data_s:0.571 smp/s:46 mem_gb:39.65
137
+ Training: 40%|β–ˆβ–ˆβ–ˆβ–ˆ | 24000/60000 [10:31:40<13:22:50, 1.34s/step]INFO 2026-07-15 11:46:40 ot_train.py:649 step:24K smpl:2M ep:35K epch:7.03 loss:0.056 grdn:0.278 lr:7.5e-05 updt_s:0.815 data_s:0.612 smp/s:45 mem_gb:39.65
138
+ Training: 40%|β–ˆβ–ˆβ–ˆβ–ˆ | 24200/60000 [10:36:13<13:32:27, 1.36s/step]INFO 2026-07-15 11:51:13 ot_train.py:649 step:24K smpl:2M ep:35K epch:7.09 loss:0.057 grdn:0.274 lr:7.5e-05 updt_s:0.806 data_s:0.556 smp/s:47 mem_gb:39.65
139
+ Training: 41%|β–ˆβ–ˆβ–ˆβ–ˆ | 24400/60000 [10:40:48<13:22:44, 1.35s/step]INFO 2026-07-15 11:55:48 ot_train.py:649 step:24K smpl:2M ep:35K epch:7.15 loss:0.055 grdn:0.267 lr:7.4e-05 updt_s:0.825 data_s:0.547 smp/s:47 mem_gb:39.65
140
+ Training: 41%|β–ˆβ–ˆβ–ˆβ–ˆ | 24600/60000 [10:45:18<13:12:07, 1.34s/step]INFO 2026-07-15 12:00:18 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.21 loss:0.056 grdn:0.262 lr:7.4e-05 updt_s:0.799 data_s:0.546 smp/s:48 mem_gb:39.65
141
+ Training: 41%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 24800/60000 [10:49:49<13:39:05, 1.40s/step]INFO 2026-07-15 12:04:50 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.27 loss:0.055 grdn:0.258 lr:7.3e-05 updt_s:0.805 data_s:0.548 smp/s:47 mem_gb:39.65
142
+ Training: 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 25000/60000 [10:54:20<13:03:59, 1.34s/step]INFO 2026-07-15 12:09:20 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.33 loss:0.055 grdn:0.270 lr:7.3e-05 updt_s:0.778 data_s:0.570 smp/s:47 mem_gb:39.63
143
+ Training: 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 25200/60000 [10:58:52<13:19:29, 1.38s/step]INFO 2026-07-15 12:13:53 ot_train.py:649 step:25K smpl:2M ep:36K epch:7.39 loss:0.053 grdn:0.258 lr:7.2e-05 updt_s:0.797 data_s:0.563 smp/s:47 mem_gb:39.65
144
+ Training: 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 25400/60000 [11:03:21<12:50:26, 1.34s/step]INFO 2026-07-15 12:18:22 ot_train.py:649 step:25K smpl:2M ep:37K epch:7.44 loss:0.052 grdn:0.253 lr:7.2e-05 updt_s:0.804 data_s:0.537 smp/s:48 mem_gb:39.65
145
+ Training: 43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 25600/60000 [11:07:57<13:50:54, 1.45s/step]INFO 2026-07-15 12:22:57 ot_train.py:649 step:26K smpl:2M ep:37K epch:7.50 loss:0.054 grdn:0.276 lr:7.1e-05 updt_s:0.810 data_s:0.561 smp/s:47 mem_gb:39.65
146
+ Training: 43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 25800/60000 [11:12:43<12:37:23, 1.33s/step]INFO 2026-07-15 12:27:44 ot_train.py:649 step:26K smpl:2M ep:37K epch:7.56 loss:0.055 grdn:0.287 lr:7.1e-05 updt_s:0.815 data_s:0.614 smp/s:45 mem_gb:39.65
147
+ Training: 43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 26000/60000 [11:17:15<12:38:36, 1.34s/step]INFO 2026-07-15 12:32:15 ot_train.py:649 step:26K smpl:2M ep:38K epch:7.62 loss:0.053 grdn:0.266 lr:7.0e-05 updt_s:0.800 data_s:0.552 smp/s:47 mem_gb:39.65
148
+ Training: 44%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 26200/60000 [11:21:44<12:28:46, 1.33s/step]INFO 2026-07-15 12:36:45 ot_train.py:649 step:26K smpl:2M ep:38K epch:7.68 loss:0.054 grdn:0.269 lr:7.0e-05 updt_s:0.798 data_s:0.546 smp/s:48 mem_gb:39.65
149
+ Training: 44%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 26400/60000 [11:26:15<12:55:37, 1.39s/step]INFO 2026-07-15 12:41:16 ot_train.py:649 step:26K smpl:2M ep:38K epch:7.74 loss:0.053 grdn:0.270 lr:6.9e-05 updt_s:0.810 data_s:0.541 smp/s:47 mem_gb:39.64
150
+ Training: 44%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 26600/60000 [11:30:46<12:17:32, 1.32s/step]INFO 2026-07-15 12:45:46 ot_train.py:649 step:27K smpl:2M ep:38K epch:7.80 loss:0.053 grdn:0.270 lr:6.8e-05 updt_s:0.804 data_s:0.544 smp/s:47 mem_gb:39.65
151
+ Training: 45%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 26800/60000 [11:35:23<14:29:56, 1.57s/step]INFO 2026-07-15 12:50:23 ot_train.py:649 step:27K smpl:2M ep:39K epch:7.85 loss:0.054 grdn:0.284 lr:6.8e-05 updt_s:0.829 data_s:0.553 smp/s:46 mem_gb:39.65
152
+ Training: 45%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27000/60000 [11:40:17<13:10:36, 1.44s/step]INFO 2026-07-15 12:55:17 ot_train.py:649 step:27K smpl:2M ep:39K epch:7.91 loss:0.052 grdn:0.266 lr:6.7e-05 updt_s:0.853 data_s:0.613 smp/s:44 mem_gb:39.65
153
+ Training: 45%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27200/60000 [11:44:57<12:19:03, 1.35s/step]INFO 2026-07-15 12:59:57 ot_train.py:649 step:27K smpl:2M ep:39K epch:7.97 loss:0.053 grdn:0.293 lr:6.7e-05 updt_s:0.828 data_s:0.570 smp/s:46 mem_gb:39.63
154
+ Training: 46%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27400/60000 [11:49:28<12:03:57, 1.33s/step]INFO 2026-07-15 13:04:28 ot_train.py:649 step:27K smpl:2M ep:40K epch:8.03 loss:0.052 grdn:0.274 lr:6.6e-05 updt_s:0.802 data_s:0.548 smp/s:47 mem_gb:39.65
155
+ Training: 46%|β–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 27600/60000 [11:53:57<11:56:14, 1.33s/step]INFO 2026-07-15 13:08:57 ot_train.py:649 step:28K smpl:2M ep:40K epch:8.09 loss:0.050 grdn:0.261 lr:6.6e-05 updt_s:0.800 data_s:0.540 smp/s:48 mem_gb:39.65
156
+ Training: 46%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 27800/60000 [11:58:26<11:54:24, 1.33s/step]INFO 2026-07-15 13:13:26 ot_train.py:649 step:28K smpl:2M ep:40K epch:8.15 loss:0.051 grdn:0.284 lr:6.5e-05 updt_s:0.801 data_s:0.542 smp/s:48 mem_gb:39.65
157
+ Training: 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 28000/60000 [12:02:55<12:06:08, 1.36s/step]INFO 2026-07-15 13:17:56 ot_train.py:649 step:28K smpl:2M ep:40K epch:8.21 loss:0.051 grdn:0.277 lr:6.5e-05 updt_s:0.799 data_s:0.545 smp/s:48 mem_gb:39.65
158
+ Training: 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 28200/60000 [12:07:29<12:42:52, 1.44s/step]INFO 2026-07-15 13:22:30 ot_train.py:649 step:28K smpl:2M ep:41K epch:8.26 loss:0.051 grdn:0.272 lr:6.4e-05 updt_s:0.818 data_s:0.547 smp/s:47 mem_gb:39.65
159
+ Training: 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 28400/60000 [12:11:59<11:39:53, 1.33s/step]INFO 2026-07-15 13:26:59 ot_train.py:649 step:28K smpl:2M ep:41K epch:8.32 loss:0.050 grdn:0.263 lr:6.4e-05 updt_s:0.806 data_s:0.539 smp/s:48 mem_gb:39.65
160
+ Training: 48%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 28600/60000 [12:16:37<12:36:49, 1.45s/step]INFO 2026-07-15 13:31:37 ot_train.py:649 step:29K smpl:2M ep:41K epch:8.38 loss:0.051 grdn:0.258 lr:6.3e-05 updt_s:0.826 data_s:0.560 smp/s:46 mem_gb:39.65
161
+ Training: 48%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 28800/60000 [12:21:42<13:28:51, 1.56s/step]INFO 2026-07-15 13:36:42 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.44 loss:0.050 grdn:0.272 lr:6.2e-05 updt_s:0.866 data_s:0.655 smp/s:42 mem_gb:39.65
162
+ Training: 48%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 29000/60000 [12:27:13<13:55:32, 1.62s/step]INFO 2026-07-15 13:42:13 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.50 loss:0.050 grdn:0.275 lr:6.2e-05 updt_s:0.916 data_s:0.731 smp/s:39 mem_gb:39.65
163
+ Training: 49%|β–ˆβ–ˆβ–ˆβ–ˆβ–Š | 29200/60000 [12:32:43<12:22:14, 1.45s/step]INFO 2026-07-15 13:47:43 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.56 loss:0.050 grdn:0.274 lr:6.1e-05 updt_s:0.876 data_s:0.768 smp/s:39 mem_gb:39.65
164
+ Training: 49%|β–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 29400/60000 [12:37:22<11:17:56, 1.33s/step]INFO 2026-07-15 13:52:23 ot_train.py:649 step:29K smpl:2M ep:42K epch:8.62 loss:0.050 grdn:0.279 lr:6.1e-05 updt_s:0.806 data_s:0.588 smp/s:46 mem_gb:39.63
165
+ Training: 49%|β–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 29600/60000 [12:41:52<11:13:16, 1.33s/step]INFO 2026-07-15 13:56:53 ot_train.py:649 step:30K smpl:2M ep:43K epch:8.68 loss:0.050 grdn:0.275 lr:6.0e-05 updt_s:0.809 data_s:0.537 smp/s:48 mem_gb:39.65
166
+ Training: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 29800/60000 [12:46:20<11:19:38, 1.35s/step]INFO 2026-07-15 14:01:20 ot_train.py:649 step:30K smpl:2M ep:43K epch:8.73 loss:0.048 grdn:0.273 lr:6.0e-05 updt_s:0.801 data_s:0.533 smp/s:48 mem_gb:39.65
167
+ Training: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30000/60000 [12:50:49<12:15:53, 1.47s/step]INFO 2026-07-15 14:05:49 ot_train.py:649 step:30K smpl:2M ep:43K epch:8.79 loss:0.048 grdn:0.270 lr:5.9e-05 updt_s:0.806 data_s:0.535 smp/s:48 mem_gb:39.65
168
+ Training: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30200/60000 [12:55:22<11:56:13, 1.44s/step]INFO 2026-07-15 14:10:22 ot_train.py:649 step:30K smpl:2M ep:44K epch:8.85 loss:0.048 grdn:0.273 lr:5.8e-05 updt_s:0.811 data_s:0.551 smp/s:47 mem_gb:39.65
169
+ Training: 51%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30400/60000 [13:00:01<12:18:10, 1.50s/step]INFO 2026-07-15 14:15:02 ot_train.py:649 step:30K smpl:2M ep:44K epch:8.91 loss:0.046 grdn:0.256 lr:5.8e-05 updt_s:0.840 data_s:0.554 smp/s:46 mem_gb:39.65
170
+ Training: 51%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 30600/60000 [13:05:08<11:42:33, 1.43s/step]INFO 2026-07-15 14:20:08 ot_train.py:649 step:31K smpl:2M ep:44K epch:8.97 loss:0.047 grdn:0.260 lr:5.7e-05 updt_s:0.897 data_s:0.630 smp/s:42 mem_gb:39.65
171
+ Training: 51%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 30800/60000 [13:10:31<12:57:37, 1.60s/step]INFO 2026-07-15 14:25:31 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.03 loss:0.047 grdn:0.273 lr:5.7e-05 updt_s:0.909 data_s:0.702 smp/s:40 mem_gb:39.65
172
+ Training: 52%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 31000/60000 [13:15:59<13:18:56, 1.65s/step]INFO 2026-07-15 14:30:59 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.09 loss:0.047 grdn:0.264 lr:5.6e-05 updt_s:0.909 data_s:0.726 smp/s:39 mem_gb:39.65
173
+ Training: 52%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 31200/60000 [13:21:20<11:55:33, 1.49s/step]INFO 2026-07-15 14:36:21 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.14 loss:0.046 grdn:0.265 lr:5.6e-05 updt_s:0.901 data_s:0.704 smp/s:40 mem_gb:39.65
174
+ Training: 52%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 31400/60000 [13:26:38<12:25:35, 1.56s/step]INFO 2026-07-15 14:41:38 ot_train.py:649 step:31K smpl:2M ep:45K epch:9.20 loss:0.049 grdn:0.291 lr:5.5e-05 updt_s:0.927 data_s:0.656 smp/s:40 mem_gb:39.65
175
+ Training: 53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 31600/60000 [13:32:05<15:29:36, 1.96s/step]INFO 2026-07-15 14:47:06 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.26 loss:0.048 grdn:0.272 lr:5.4e-05 updt_s:0.893 data_s:0.739 smp/s:39 mem_gb:39.63
176
+ Training: 53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 31800/60000 [13:37:36<12:42:22, 1.62s/step]INFO 2026-07-15 14:52:36 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.32 loss:0.045 grdn:0.267 lr:5.4e-05 updt_s:0.886 data_s:0.761 smp/s:39 mem_gb:39.65
177
+ Training: 53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 32000/60000 [13:43:08<12:59:36, 1.67s/step]INFO 2026-07-15 14:58:09 ot_train.py:649 step:32K smpl:2M ep:46K epch:9.38 loss:0.045 grdn:0.264 lr:5.3e-05 updt_s:0.891 data_s:0.769 smp/s:39 mem_gb:39.65
178
+ Training: 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 32200/60000 [13:48:42<12:49:20, 1.66s/step]INFO 2026-07-15 15:03:42 ot_train.py:649 step:32K smpl:2M ep:47K epch:9.44 loss:0.045 grdn:0.266 lr:5.3e-05 updt_s:0.894 data_s:0.767 smp/s:39 mem_gb:39.65
179
+ Training: 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 32400/60000 [13:54:14<13:26:32, 1.75s/step]INFO 2026-07-15 15:09:15 ot_train.py:649 step:32K smpl:2M ep:47K epch:9.50 loss:0.045 grdn:0.270 lr:5.2e-05 updt_s:0.886 data_s:0.771 smp/s:39 mem_gb:39.65
180
+ Training: 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 32600/60000 [13:59:44<12:05:42, 1.59s/step]INFO 2026-07-15 15:14:44 ot_train.py:649 step:33K smpl:2M ep:47K epch:9.55 loss:0.045 grdn:0.264 lr:5.1e-05 updt_s:0.891 data_s:0.753 smp/s:39 mem_gb:39.65
181
+ Training: 55%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 32800/60000 [14:05:16<13:15:31, 1.75s/step]INFO 2026-07-15 15:20:16 ot_train.py:649 step:33K smpl:2M ep:47K epch:9.61 loss:0.046 grdn:0.276 lr:5.1e-05 updt_s:0.907 data_s:0.747 smp/s:39 mem_gb:39.65
182
+ Training: 55%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33000/60000 [14:10:46<12:09:18, 1.62s/step]INFO 2026-07-15 15:25:47 ot_train.py:649 step:33K smpl:2M ep:48K epch:9.67 loss:0.045 grdn:0.270 lr:5.0e-05 updt_s:0.884 data_s:0.765 smp/s:39 mem_gb:39.65
183
+ Training: 55%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33200/60000 [14:16:19<12:15:16, 1.65s/step]INFO 2026-07-15 15:31:19 ot_train.py:649 step:33K smpl:2M ep:48K epch:9.73 loss:0.044 grdn:0.268 lr:5.0e-05 updt_s:0.901 data_s:0.755 smp/s:39 mem_gb:39.65
184
+ Training: 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33400/60000 [14:21:47<12:15:57, 1.66s/step]INFO 2026-07-15 15:36:48 ot_train.py:649 step:33K smpl:2M ep:48K epch:9.79 loss:0.044 grdn:0.268 lr:4.9e-05 updt_s:0.895 data_s:0.744 smp/s:39 mem_gb:39.65
185
+ Training: 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 33600/60000 [14:27:19<12:05:29, 1.65s/step]INFO 2026-07-15 15:42:19 ot_train.py:649 step:34K smpl:2M ep:49K epch:9.85 loss:0.045 grdn:0.280 lr:4.9e-05 updt_s:0.897 data_s:0.756 smp/s:39 mem_gb:39.65
186
+ Training: 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 33800/60000 [14:32:50<11:58:31, 1.65s/step]INFO 2026-07-15 15:47:51 ot_train.py:649 step:34K smpl:2M ep:49K epch:9.91 loss:0.045 grdn:0.267 lr:4.8e-05 updt_s:0.900 data_s:0.751 smp/s:39 mem_gb:39.65
187
+ Training: 57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 34000/60000 [14:38:20<11:58:31, 1.66s/step]INFO 2026-07-15 15:53:21 ot_train.py:649 step:34K smpl:2M ep:49K epch:9.96 loss:0.043 grdn:0.264 lr:4.7e-05 updt_s:0.891 data_s:0.754 smp/s:39 mem_gb:39.63
188
+ Training: 57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 34200/60000 [14:43:50<11:59:06, 1.67s/step]INFO 2026-07-15 15:58:50 ot_train.py:649 step:34K smpl:2M ep:49K epch:10.02 loss:0.042 grdn:0.266 lr:4.7e-05 updt_s:0.885 data_s:0.757 smp/s:39 mem_gb:39.65
189
+ Training: 57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 34400/60000 [14:49:21<12:00:19, 1.69s/step]INFO 2026-07-15 16:04:21 ot_train.py:649 step:34K smpl:2M ep:50K epch:10.08 loss:0.042 grdn:0.275 lr:4.6e-05 updt_s:0.918 data_s:0.733 smp/s:39 mem_gb:39.65
190
+ Training: 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 34600/60000 [14:54:50<11:54:54, 1.69s/step]INFO 2026-07-15 16:09:50 ot_train.py:649 step:35K smpl:2M ep:50K epch:10.14 loss:0.041 grdn:0.258 lr:4.6e-05 updt_s:0.874 data_s:0.766 smp/s:39 mem_gb:39.65
191
+ Training: 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 34800/60000 [15:00:22<11:38:48, 1.66s/step]INFO 2026-07-15 16:15:22 ot_train.py:649 step:35K smpl:2M ep:50K epch:10.20 loss:0.042 grdn:0.265 lr:4.5e-05 updt_s:0.899 data_s:0.756 smp/s:39 mem_gb:39.65
192
+ Training: 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 35000/60000 [15:05:53<12:08:58, 1.75s/step]INFO 2026-07-15 16:20:53 ot_train.py:649 step:35K smpl:2M ep:51K epch:10.26 loss:0.042 grdn:0.270 lr:4.4e-05 updt_s:0.883 data_s:0.765 smp/s:39 mem_gb:39.65
193
+ Training: 59%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 35200/60000 [15:11:27<11:40:05, 1.69s/step]INFO 2026-07-15 16:26:27 ot_train.py:649 step:35K smpl:2M ep:51K epch:10.32 loss:0.043 grdn:0.288 lr:4.4e-05 updt_s:0.879 data_s:0.788 smp/s:38 mem_gb:39.65
194
+ Training: 59%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 35400/60000 [15:17:04<11:11:25, 1.64s/step]INFO 2026-07-15 16:32:05 ot_train.py:649 step:35K smpl:2M ep:51K epch:10.38 loss:0.041 grdn:0.274 lr:4.3e-05 updt_s:0.915 data_s:0.767 smp/s:38 mem_gb:39.65
195
+ Training: 59%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 35600/60000 [15:22:37<10:43:46, 1.58s/step]INFO 2026-07-15 16:37:37 ot_train.py:649 step:36K smpl:2M ep:51K epch:10.43 loss:0.041 grdn:0.269 lr:4.3e-05 updt_s:0.899 data_s:0.759 smp/s:39 mem_gb:39.65
196
+ Training: 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 35800/60000 [15:28:05<11:01:21, 1.64s/step]INFO 2026-07-15 16:43:05 ot_train.py:649 step:36K smpl:2M ep:52K epch:10.49 loss:0.041 grdn:0.279 lr:4.2e-05 updt_s:0.901 data_s:0.734 smp/s:39 mem_gb:39.65
197
+ Training: 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36000/60000 [15:33:36<10:51:07, 1.63s/step]INFO 2026-07-15 16:48:36 ot_train.py:649 step:36K smpl:2M ep:52K epch:10.55 loss:0.041 grdn:0.275 lr:4.2e-05 updt_s:0.914 data_s:0.735 smp/s:39 mem_gb:39.65
198
+ Training: 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36200/60000 [15:39:07<11:01:45, 1.67s/step]INFO 2026-07-15 16:54:07 ot_train.py:649 step:36K smpl:2M ep:52K epch:10.61 loss:0.040 grdn:0.273 lr:4.1e-05 updt_s:0.909 data_s:0.741 smp/s:39 mem_gb:39.63
199
+ Training: 61%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36400/60000 [15:44:43<11:13:28, 1.71s/step]INFO 2026-07-15 16:59:43 ot_train.py:649 step:36K smpl:2M ep:53K epch:10.67 loss:0.040 grdn:0.275 lr:4.0e-05 updt_s:0.932 data_s:0.742 smp/s:38 mem_gb:39.65
200
+ Training: 61%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36600/60000 [15:50:14<10:52:18, 1.67s/step]INFO 2026-07-15 17:05:14 ot_train.py:649 step:37K smpl:2M ep:53K epch:10.73 loss:0.039 grdn:0.279 lr:4.0e-05 updt_s:0.921 data_s:0.729 smp/s:39 mem_gb:39.65
201
+ Training: 61%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 36800/60000 [15:55:53<10:54:37, 1.69s/step]INFO 2026-07-15 17:10:53 ot_train.py:649 step:37K smpl:2M ep:53K epch:10.79 loss:0.038 grdn:0.275 lr:3.9e-05 updt_s:0.927 data_s:0.763 smp/s:38 mem_gb:39.65
202
+ Training: 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 37000/60000 [16:01:20<9:57:23, 1.56s/step] INFO 2026-07-15 17:16:21 ot_train.py:649 step:37K smpl:2M ep:53K epch:10.84 loss:0.038 grdn:0.273 lr:3.9e-05 updt_s:0.864 data_s:0.768 smp/s:39 mem_gb:39.65
203
+ Training: 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 37200/60000 [16:06:54<10:18:29, 1.63s/step]INFO 2026-07-15 17:21:55 ot_train.py:649 step:37K smpl:2M ep:54K epch:10.90 loss:0.039 grdn:0.273 lr:3.8e-05 updt_s:0.906 data_s:0.760 smp/s:38 mem_gb:39.65
204
+ Training: 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 37400/60000 [16:12:25<10:34:56, 1.69s/step]INFO 2026-07-15 17:27:25 ot_train.py:649 step:37K smpl:2M ep:54K epch:10.96 loss:0.039 grdn:0.268 lr:3.8e-05 updt_s:0.899 data_s:0.749 smp/s:39 mem_gb:39.65
205
+ Training: 63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 37600/60000 [16:17:55<10:17:34, 1.65s/step]INFO 2026-07-15 17:32:56 ot_train.py:649 step:38K smpl:2M ep:54K epch:11.02 loss:0.038 grdn:0.278 lr:3.7e-05 updt_s:0.899 data_s:0.748 smp/s:39 mem_gb:39.65
206
+ Training: 63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 37800/60000 [16:23:30<10:07:20, 1.64s/step]INFO 2026-07-15 17:38:31 ot_train.py:649 step:38K smpl:2M ep:55K epch:11.08 loss:0.039 grdn:0.283 lr:3.6e-05 updt_s:0.902 data_s:0.768 smp/s:38 mem_gb:39.65
207
+ Training: 63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 38000/60000 [16:29:02<10:04:25, 1.65s/step]INFO 2026-07-15 17:44:02 ot_train.py:649 step:38K smpl:2M ep:55K epch:11.14 loss:0.038 grdn:0.279 lr:3.6e-05 updt_s:0.919 data_s:0.733 smp/s:39 mem_gb:39.65
208
+ Training: 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 38200/60000 [16:34:34<10:26:43, 1.72s/step]INFO 2026-07-15 17:49:34 ot_train.py:649 step:38K smpl:2M ep:55K epch:11.20 loss:0.039 grdn:0.282 lr:3.5e-05 updt_s:0.896 data_s:0.757 smp/s:39 mem_gb:39.65
209
+ Training: 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 38400/60000 [16:40:07<9:43:40, 1.62s/step]INFO 2026-07-15 17:55:07 ot_train.py:649 step:38K smpl:2M ep:55K epch:11.25 loss:0.039 grdn:0.282 lr:3.5e-05 updt_s:0.898 data_s:0.764 smp/s:39 mem_gb:39.63
210
+ Training: 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 38600/60000 [16:45:41<9:59:21, 1.68s/step] INFO 2026-07-15 18:00:42 ot_train.py:649 step:39K smpl:2M ep:56K epch:11.31 loss:0.036 grdn:0.271 lr:3.4e-05 updt_s:0.898 data_s:0.769 smp/s:38 mem_gb:39.65
211
+ Training: 65%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 38800/60000 [16:51:14<9:42:48, 1.65s/step]INFO 2026-07-15 18:06:15 ot_train.py:649 step:39K smpl:2M ep:56K epch:11.37 loss:0.036 grdn:0.274 lr:3.4e-05 updt_s:0.887 data_s:0.773 smp/s:39 mem_gb:39.65
212
+ Training: 65%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39000/60000 [16:56:47<8:56:40, 1.53s/step]INFO 2026-07-15 18:11:47 ot_train.py:649 step:39K smpl:2M ep:56K epch:11.43 loss:0.036 grdn:0.277 lr:3.3e-05 updt_s:0.893 data_s:0.763 smp/s:39 mem_gb:39.65
213
+ Training: 65%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39200/60000 [17:02:13<9:34:09, 1.66s/step]INFO 2026-07-15 18:17:14 ot_train.py:649 step:39K smpl:3M ep:57K epch:11.49 loss:0.036 grdn:0.265 lr:3.3e-05 updt_s:0.903 data_s:0.726 smp/s:39 mem_gb:39.65
214
+ Training: 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39400/60000 [17:07:46<9:30:10, 1.66s/step]INFO 2026-07-15 18:22:46 ot_train.py:649 step:39K smpl:3M ep:57K epch:11.55 loss:0.035 grdn:0.284 lr:3.2e-05 updt_s:0.911 data_s:0.745 smp/s:39 mem_gb:39.65
215
+ Training: 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 39600/60000 [17:13:17<9:28:58, 1.67s/step]INFO 2026-07-15 18:28:18 ot_train.py:649 step:40K smpl:3M ep:57K epch:11.61 loss:0.036 grdn:0.280 lr:3.2e-05 updt_s:0.902 data_s:0.751 smp/s:39 mem_gb:39.65
216
+ Training: 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 39800/60000 [17:18:48<9:09:37, 1.63s/step]INFO 2026-07-15 18:33:49 ot_train.py:649 step:40K smpl:3M ep:58K epch:11.66 loss:0.035 grdn:0.272 lr:3.1e-05 updt_s:0.879 data_s:0.772 smp/s:39 mem_gb:39.65
217
+ Training: 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 40000/60000 [17:24:21<9:41:26, 1.74s/step]INFO 2026-07-15 18:39:21 ot_train.py:649 step:40K smpl:3M ep:58K epch:11.72 loss:0.036 grdn:0.301 lr:3.0e-05 updt_s:0.908 data_s:0.748 smp/s:39 mem_gb:39.65
218
+ Training: 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 40200/60000 [17:29:52<9:08:19, 1.66s/step]INFO 2026-07-15 18:44:52 ot_train.py:649 step:40K smpl:3M ep:58K epch:11.78 loss:0.035 grdn:0.285 lr:3.0e-05 updt_s:0.882 data_s:0.769 smp/s:39 mem_gb:39.65
219
+ Training: 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 40400/60000 [17:35:22<8:29:03, 1.56s/step]INFO 2026-07-15 18:50:22 ot_train.py:649 step:40K smpl:3M ep:58K epch:11.84 loss:0.035 grdn:0.289 lr:2.9e-05 updt_s:0.883 data_s:0.762 smp/s:39 mem_gb:39.65
220
+ Training: 68%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 40600/60000 [17:40:51<8:55:50, 1.66s/step]INFO 2026-07-15 18:55:52 ot_train.py:649 step:41K smpl:3M ep:59K epch:11.90 loss:0.036 grdn:0.274 lr:2.9e-05 updt_s:0.880 data_s:0.762 smp/s:39 mem_gb:39.65
221
+ Training: 68%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 40800/60000 [17:46:23<8:22:22, 1.57s/step]INFO 2026-07-15 19:01:23 ot_train.py:649 step:41K smpl:3M ep:59K epch:11.96 loss:0.033 grdn:0.284 lr:2.8e-05 updt_s:0.875 data_s:0.779 smp/s:39 mem_gb:39.63
222
+ Training: 68%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 41000/60000 [17:51:55<8:46:14, 1.66s/step]INFO 2026-07-15 19:06:56 ot_train.py:649 step:41K smpl:3M ep:59K epch:12.02 loss:0.033 grdn:0.279 lr:2.8e-05 updt_s:0.884 data_s:0.773 smp/s:39 mem_gb:39.65
223
+ Training: 69%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 41200/60000 [17:57:29<8:53:16, 1.70s/step]INFO 2026-07-15 19:12:30 ot_train.py:649 step:41K smpl:3M ep:60K epch:12.08 loss:0.033 grdn:0.283 lr:2.7e-05 updt_s:0.890 data_s:0.774 smp/s:38 mem_gb:39.65
224
+ Training: 69%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 41400/60000 [18:03:04<8:42:38, 1.69s/step]INFO 2026-07-15 19:18:04 ot_train.py:649 step:41K smpl:3M ep:60K epch:12.13 loss:0.033 grdn:0.283 lr:2.7e-05 updt_s:0.889 data_s:0.779 smp/s:38 mem_gb:39.65
225
+ Training: 69%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 41600/60000 [18:08:39<9:24:26, 1.84s/step]INFO 2026-07-15 19:23:40 ot_train.py:649 step:42K smpl:3M ep:60K epch:12.19 loss:0.033 grdn:0.278 lr:2.6e-05 updt_s:0.894 data_s:0.780 smp/s:38 mem_gb:39.65
226
+ Training: 70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 41800/60000 [18:14:12<8:32:23, 1.69s/step]INFO 2026-07-15 19:29:12 ot_train.py:649 step:42K smpl:3M ep:60K epch:12.25 loss:0.033 grdn:0.281 lr:2.6e-05 updt_s:0.868 data_s:0.790 smp/s:39 mem_gb:39.65
227
+ Training: 70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42000/60000 [18:19:51<8:13:53, 1.65s/step]INFO 2026-07-15 19:34:51 ot_train.py:649 step:42K smpl:3M ep:61K epch:12.31 loss:0.033 grdn:0.281 lr:2.5e-05 updt_s:0.895 data_s:0.795 smp/s:38 mem_gb:39.65
228
+ Training: 70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42200/60000 [18:25:24<9:48:29, 1.98s/step]INFO 2026-07-15 19:40:24 ot_train.py:649 step:42K smpl:3M ep:61K epch:12.37 loss:0.033 grdn:0.273 lr:2.5e-05 updt_s:0.910 data_s:0.748 smp/s:39 mem_gb:39.65
229
+ Training: 71%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42400/60000 [18:31:00<8:14:29, 1.69s/step]INFO 2026-07-15 19:46:00 ot_train.py:649 step:42K smpl:3M ep:61K epch:12.43 loss:0.033 grdn:0.287 lr:2.4e-05 updt_s:0.908 data_s:0.768 smp/s:38 mem_gb:39.65
230
+ Training: 71%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 42600/60000 [18:36:31<8:07:33, 1.68s/step]INFO 2026-07-15 19:51:32 ot_train.py:649 step:43K smpl:3M ep:62K epch:12.49 loss:0.033 grdn:0.275 lr:2.4e-05 updt_s:0.865 data_s:0.787 smp/s:39 mem_gb:39.65
231
+ Training: 71%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 42800/60000 [18:42:03<8:02:53, 1.68s/step]INFO 2026-07-15 19:57:03 ot_train.py:649 step:43K smpl:3M ep:62K epch:12.54 loss:0.032 grdn:0.283 lr:2.3e-05 updt_s:0.873 data_s:0.779 smp/s:39 mem_gb:39.65
232
+ Training: 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 43000/60000 [18:47:39<8:03:43, 1.71s/step]INFO 2026-07-15 20:02:39 ot_train.py:649 step:43K smpl:3M ep:62K epch:12.60 loss:0.031 grdn:0.270 lr:2.3e-05 updt_s:0.908 data_s:0.768 smp/s:38 mem_gb:39.63
233
+ Training: 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 43200/60000 [18:53:13<7:30:45, 1.61s/step]INFO 2026-07-15 20:08:13 ot_train.py:649 step:43K smpl:3M ep:62K epch:12.66 loss:0.030 grdn:0.276 lr:2.2e-05 updt_s:0.894 data_s:0.770 smp/s:38 mem_gb:39.65
234
+ Training: 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 43400/60000 [18:58:43<7:25:35, 1.61s/step]INFO 2026-07-15 20:13:43 ot_train.py:649 step:43K smpl:3M ep:63K epch:12.72 loss:0.031 grdn:0.273 lr:2.2e-05 updt_s:0.897 data_s:0.748 smp/s:39 mem_gb:39.65
235
+ Training: 73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 43600/60000 [19:04:15<7:41:36, 1.69s/step]INFO 2026-07-15 20:19:15 ot_train.py:649 step:44K smpl:3M ep:63K epch:12.78 loss:0.030 grdn:0.278 lr:2.1e-05 updt_s:0.888 data_s:0.769 smp/s:39 mem_gb:39.65
236
+ Training: 73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 43800/60000 [19:09:46<7:19:43, 1.63s/step]INFO 2026-07-15 20:24:46 ot_train.py:649 step:44K smpl:3M ep:63K epch:12.84 loss:0.030 grdn:0.286 lr:2.1e-05 updt_s:0.880 data_s:0.769 smp/s:39 mem_gb:39.65
237
+ Training: 73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 44000/60000 [19:15:19<7:19:15, 1.65s/step]INFO 2026-07-15 20:30:20 ot_train.py:649 step:44K smpl:3M ep:64K epch:12.90 loss:0.030 grdn:0.279 lr:2.0e-05 updt_s:0.872 data_s:0.791 smp/s:38 mem_gb:39.65
238
+ Training: 74%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 44200/60000 [19:20:53<7:13:49, 1.65s/step]INFO 2026-07-15 20:35:53 ot_train.py:649 step:44K smpl:3M ep:64K epch:12.95 loss:0.030 grdn:0.272 lr:2.0e-05 updt_s:0.888 data_s:0.777 smp/s:38 mem_gb:39.65
239
+ Training: 74%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 44400/60000 [19:26:25<7:15:33, 1.68s/step]INFO 2026-07-15 20:41:26 ot_train.py:649 step:44K smpl:3M ep:64K epch:13.01 loss:0.030 grdn:0.284 lr:1.9e-05 updt_s:0.887 data_s:0.770 smp/s:39 mem_gb:39.65
240
+ Training: 74%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 44600/60000 [19:31:57<7:00:20, 1.64s/step]INFO 2026-07-15 20:46:57 ot_train.py:649 step:45K smpl:3M ep:64K epch:13.07 loss:0.029 grdn:0.275 lr:1.9e-05 updt_s:0.872 data_s:0.780 smp/s:39 mem_gb:39.65
241
+ Training: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 44800/60000 [19:37:28<6:54:07, 1.63s/step]INFO 2026-07-15 20:52:28 ot_train.py:649 step:45K smpl:3M ep:65K epch:13.13 loss:0.029 grdn:0.265 lr:1.9e-05 updt_s:0.864 data_s:0.786 smp/s:39 mem_gb:39.65
242
+ Training: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45000/60000 [19:43:01<6:48:42, 1.63s/step]INFO 2026-07-15 20:58:01 ot_train.py:649 step:45K smpl:3M ep:65K epch:13.19 loss:0.030 grdn:0.279 lr:1.8e-05 updt_s:0.884 data_s:0.777 smp/s:39 mem_gb:39.65
243
+ Training: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45200/60000 [19:48:32<6:52:14, 1.67s/step]INFO 2026-07-15 21:03:33 ot_train.py:649 step:45K smpl:3M ep:65K epch:13.25 loss:0.029 grdn:0.269 lr:1.8e-05 updt_s:0.857 data_s:0.793 smp/s:39 mem_gb:39.63
244
+ Training: 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45400/60000 [19:54:04<6:52:36, 1.70s/step]INFO 2026-07-15 21:09:04 ot_train.py:649 step:45K smpl:3M ep:66K epch:13.31 loss:0.029 grdn:0.282 lr:1.7e-05 updt_s:0.874 data_s:0.779 smp/s:39 mem_gb:39.65
245
+ Training: 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 45600/60000 [19:59:37<6:44:32, 1.69s/step]INFO 2026-07-15 21:14:37 ot_train.py:649 step:46K smpl:3M ep:66K epch:13.36 loss:0.028 grdn:0.273 lr:1.7e-05 updt_s:0.885 data_s:0.775 smp/s:39 mem_gb:39.65
246
+ Training: 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 45800/60000 [20:05:10<6:27:43, 1.64s/step]INFO 2026-07-15 21:20:10 ot_train.py:649 step:46K smpl:3M ep:66K epch:13.42 loss:0.028 grdn:0.280 lr:1.6e-05 updt_s:0.872 data_s:0.790 smp/s:39 mem_gb:39.65
247
+ Training: 77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 46000/60000 [20:10:40<6:30:17, 1.67s/step]INFO 2026-07-15 21:25:41 ot_train.py:649 step:46K smpl:3M ep:66K epch:13.48 loss:0.027 grdn:0.278 lr:1.6e-05 updt_s:0.871 data_s:0.775 smp/s:39 mem_gb:39.65
248
+ Training: 77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 46200/60000 [20:16:12<6:14:10, 1.63s/step]INFO 2026-07-15 21:31:13 ot_train.py:649 step:46K smpl:3M ep:67K epch:13.54 loss:0.028 grdn:0.270 lr:1.5e-05 updt_s:0.882 data_s:0.774 smp/s:39 mem_gb:39.65
249
+ Training: 77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 46400/60000 [20:21:45<6:42:41, 1.78s/step]INFO 2026-07-15 21:36:45 ot_train.py:649 step:46K smpl:3M ep:67K epch:13.60 loss:0.028 grdn:0.284 lr:1.5e-05 updt_s:0.889 data_s:0.769 smp/s:39 mem_gb:39.65
250
+ Training: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 46600/60000 [20:27:17<5:58:05, 1.60s/step]INFO 2026-07-15 21:42:18 ot_train.py:649 step:47K smpl:3M ep:67K epch:13.66 loss:0.027 grdn:0.278 lr:1.5e-05 updt_s:0.895 data_s:0.762 smp/s:39 mem_gb:39.65
251
+ Training: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 46800/60000 [20:32:42<6:00:52, 1.64s/step]INFO 2026-07-15 21:47:42 ot_train.py:649 step:47K smpl:3M ep:68K epch:13.72 loss:0.027 grdn:0.281 lr:1.4e-05 updt_s:0.924 data_s:0.692 smp/s:40 mem_gb:39.65
252
+ Training: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 47000/60000 [20:38:13<6:04:53, 1.68s/step]INFO 2026-07-15 21:53:13 ot_train.py:649 step:47K smpl:3M ep:68K epch:13.77 loss:0.028 grdn:0.305 lr:1.4e-05 updt_s:0.898 data_s:0.753 smp/s:39 mem_gb:39.65
253
+ Training: 79%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 47200/60000 [20:43:27<5:31:43, 1.55s/step]INFO 2026-07-15 21:58:28 ot_train.py:649 step:47K smpl:3M ep:68K epch:13.83 loss:0.028 grdn:0.278 lr:1.3e-05 updt_s:0.913 data_s:0.654 smp/s:41 mem_gb:39.65
254
+ Training: 79%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 47400/60000 [20:48:42<5:23:56, 1.54s/step]INFO 2026-07-15 22:03:42 ot_train.py:649 step:47K smpl:3M ep:68K epch:13.89 loss:0.027 grdn:0.268 lr:1.3e-05 updt_s:0.970 data_s:0.598 smp/s:41 mem_gb:39.63
255
+ Training: 79%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 47600/60000 [20:53:46<5:00:27, 1.45s/step]INFO 2026-07-15 22:08:46 ot_train.py:649 step:48K smpl:3M ep:69K epch:13.95 loss:0.026 grdn:0.262 lr:1.3e-05 updt_s:0.964 data_s:0.555 smp/s:42 mem_gb:39.65
256
+ Training: 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 47800/60000 [20:58:52<5:07:57, 1.51s/step]INFO 2026-07-15 22:13:52 ot_train.py:649 step:48K smpl:3M ep:69K epch:14.01 loss:0.026 grdn:0.267 lr:1.2e-05 updt_s:0.965 data_s:0.561 smp/s:42 mem_gb:39.65
257
+ Training: 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48000/60000 [21:03:57<5:17:02, 1.59s/step]INFO 2026-07-15 22:18:57 ot_train.py:649 step:48K smpl:3M ep:69K epch:14.07 loss:0.026 grdn:0.275 lr:1.2e-05 updt_s:0.963 data_s:0.559 smp/s:42 mem_gb:39.65
258
+ Training: 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48200/60000 [21:09:09<5:16:35, 1.61s/step]INFO 2026-07-15 22:24:09 ot_train.py:649 step:48K smpl:3M ep:70K epch:14.13 loss:0.026 grdn:0.273 lr:1.2e-05 updt_s:0.978 data_s:0.578 smp/s:41 mem_gb:39.65
259
+ Training: 81%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48400/60000 [21:14:20<4:55:40, 1.53s/step]INFO 2026-07-15 22:29:20 ot_train.py:649 step:48K smpl:3M ep:70K epch:14.19 loss:0.026 grdn:0.267 lr:1.1e-05 updt_s:0.933 data_s:0.617 smp/s:41 mem_gb:39.65
260
+ Training: 81%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 48600/60000 [21:19:50<5:11:16, 1.64s/step]INFO 2026-07-15 22:34:51 ot_train.py:649 step:49K smpl:3M ep:70K epch:14.24 loss:0.025 grdn:0.260 lr:1.1e-05 updt_s:0.894 data_s:0.754 smp/s:39 mem_gb:39.65
261
+ Training: 81%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 48800/60000 [21:25:23<5:06:32, 1.64s/step]INFO 2026-07-15 22:40:23 ot_train.py:649 step:49K smpl:3M ep:71K epch:14.30 loss:0.026 grdn:0.277 lr:1.0e-05 updt_s:0.866 data_s:0.792 smp/s:39 mem_gb:39.65
262
+ Training: 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 49000/60000 [21:30:57<5:02:20, 1.65s/step]INFO 2026-07-15 22:45:57 ot_train.py:649 step:49K smpl:3M ep:71K epch:14.36 loss:0.025 grdn:0.260 lr:1.0e-05 updt_s:0.874 data_s:0.792 smp/s:38 mem_gb:39.65
263
+ Training: 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 49200/60000 [21:36:29<5:00:34, 1.67s/step]INFO 2026-07-15 22:51:29 ot_train.py:649 step:49K smpl:3M ep:71K epch:14.42 loss:0.025 grdn:0.268 lr:9.7e-06 updt_s:0.862 data_s:0.794 smp/s:39 mem_gb:39.65
264
+ Training: 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 49400/60000 [21:42:02<4:53:06, 1.66s/step]INFO 2026-07-15 22:57:03 ot_train.py:649 step:49K smpl:3M ep:71K epch:14.48 loss:0.025 grdn:0.261 lr:9.4e-06 updt_s:0.862 data_s:0.800 smp/s:39 mem_gb:39.65
265
+ Training: 83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 49600/60000 [21:47:35<4:47:52, 1.66s/step]INFO 2026-07-15 23:02:35 ot_train.py:649 step:50K smpl:3M ep:72K epch:14.54 loss:0.025 grdn:0.276 lr:9.0e-06 updt_s:0.855 data_s:0.802 smp/s:39 mem_gb:39.65
266
+ Training: 83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 49800/60000 [21:53:06<4:39:57, 1.65s/step]INFO 2026-07-15 23:08:07 ot_train.py:649 step:50K smpl:3M ep:72K epch:14.60 loss:0.024 grdn:0.263 lr:8.7e-06 updt_s:0.848 data_s:0.805 smp/s:39 mem_gb:39.63
267
+ Training: 83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 50000/60000 [21:58:35<4:33:42, 1.64s/step]INFO 2026-07-15 23:13:35 ot_train.py:649 step:50K smpl:3M ep:72K epch:14.65 loss:0.024 grdn:0.271 lr:8.4e-06 updt_s:0.855 data_s:0.785 smp/s:39 mem_gb:39.65
268
+ Training: 84%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 50200/60000 [22:04:13<4:59:35, 1.83s/step]INFO 2026-07-15 23:19:14 ot_train.py:649 step:50K smpl:3M ep:73K epch:14.71 loss:0.024 grdn:0.259 lr:8.1e-06 updt_s:0.891 data_s:0.796 smp/s:38 mem_gb:39.65
269
+ Training: 84%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 50400/60000 [22:09:46<4:32:17, 1.70s/step]INFO 2026-07-15 23:24:46 ot_train.py:649 step:50K smpl:3M ep:73K epch:14.77 loss:0.024 grdn:0.262 lr:7.8e-06 updt_s:0.869 data_s:0.788 smp/s:39 mem_gb:39.65
270
+ Training: 84%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 50600/60000 [22:15:18<4:28:31, 1.71s/step]INFO 2026-07-15 23:30:18 ot_train.py:649 step:51K smpl:3M ep:73K epch:14.83 loss:0.025 grdn:0.270 lr:7.4e-06 updt_s:0.858 data_s:0.799 smp/s:39 mem_gb:39.65
271
+ Training: 85%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 50800/60000 [22:20:39<4:00:08, 1.57s/step]INFO 2026-07-15 23:35:40 ot_train.py:649 step:51K smpl:3M ep:73K epch:14.89 loss:0.024 grdn:0.265 lr:7.1e-06 updt_s:0.884 data_s:0.717 smp/s:40 mem_gb:39.65
272
+ Training: 85%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 51000/60000 [22:25:46<3:38:45, 1.46s/step]INFO 2026-07-15 23:40:46 ot_train.py:649 step:51K smpl:3M ep:74K epch:14.95 loss:0.023 grdn:0.252 lr:6.8e-06 updt_s:0.893 data_s:0.637 smp/s:42 mem_gb:39.65
273
+ Training: 85%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 51200/60000 [22:31:21<3:53:05, 1.59s/step]INFO 2026-07-15 23:46:22 ot_train.py:649 step:51K smpl:3M ep:74K epch:15.01 loss:0.024 grdn:0.266 lr:6.6e-06 updt_s:0.948 data_s:0.723 smp/s:38 mem_gb:39.65
274
+ Training: 86%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 51400/60000 [22:36:54<3:59:22, 1.67s/step]INFO 2026-07-15 23:51:54 ot_train.py:649 step:51K smpl:3M ep:74K epch:15.06 loss:0.024 grdn:0.264 lr:6.3e-06 updt_s:0.914 data_s:0.745 smp/s:39 mem_gb:39.65
275
+ Training: 86%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 51600/60000 [22:42:22<3:44:11, 1.60s/step]INFO 2026-07-15 23:57:23 ot_train.py:649 step:52K smpl:3M ep:75K epch:15.12 loss:0.023 grdn:0.272 lr:6.0e-06 updt_s:0.922 data_s:0.717 smp/s:39 mem_gb:39.65
276
+ Training: 86%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 51800/60000 [22:47:52<3:55:53, 1.73s/step]INFO 2026-07-16 00:02:52 ot_train.py:649 step:52K smpl:3M ep:75K epch:15.18 loss:0.024 grdn:0.259 lr:5.7e-06 updt_s:0.906 data_s:0.737 smp/s:39 mem_gb:39.65
277
+ Training: 87%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 52000/60000 [22:53:28<3:36:46, 1.63s/step]INFO 2026-07-16 00:08:28 ot_train.py:649 step:52K smpl:3M ep:75K epch:15.24 loss:0.024 grdn:0.266 lr:5.4e-06 updt_s:0.907 data_s:0.766 smp/s:38 mem_gb:39.63
278
+ Training: 87%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 52200/60000 [22:58:59<3:42:30, 1.71s/step]INFO 2026-07-16 00:13:59 ot_train.py:649 step:52K smpl:3M ep:75K epch:15.30 loss:0.023 grdn:0.264 lr:5.2e-06 updt_s:0.911 data_s:0.740 smp/s:39 mem_gb:39.65
279
+ Training: 87%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 52400/60000 [23:04:31<3:21:53, 1.59s/step]INFO 2026-07-16 00:19:31 ot_train.py:649 step:52K smpl:3M ep:76K epch:15.36 loss:0.023 grdn:0.237 lr:4.9e-06 updt_s:0.883 data_s:0.774 smp/s:39 mem_gb:39.65
280
+ Training: 88%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 52600/60000 [23:10:06<3:29:16, 1.70s/step]INFO 2026-07-16 00:25:07 ot_train.py:649 step:53K smpl:3M ep:76K epch:15.42 loss:0.023 grdn:0.238 lr:4.7e-06 updt_s:0.893 data_s:0.778 smp/s:38 mem_gb:39.65
281
+ Training: 88%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 52800/60000 [23:15:41<3:22:36, 1.69s/step]INFO 2026-07-16 00:30:41 ot_train.py:649 step:53K smpl:3M ep:76K epch:15.47 loss:0.023 grdn:0.251 lr:4.4e-06 updt_s:0.889 data_s:0.780 smp/s:38 mem_gb:39.65
282
+ Training: 88%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 53000/60000 [23:21:23<3:09:06, 1.62s/step]INFO 2026-07-16 00:36:23 ot_train.py:649 step:53K smpl:3M ep:77K epch:15.53 loss:0.022 grdn:0.251 lr:4.2e-06 updt_s:0.909 data_s:0.795 smp/s:38 mem_gb:39.65
283
+ Training: 89%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 53200/60000 [23:26:56<3:08:02, 1.66s/step]INFO 2026-07-16 00:41:57 ot_train.py:649 step:53K smpl:3M ep:77K epch:15.59 loss:0.022 grdn:0.264 lr:4.0e-06 updt_s:0.886 data_s:0.776 smp/s:39 mem_gb:39.65
284
+ Training: 89%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 53400/60000 [23:32:31<3:16:43, 1.79s/step]INFO 2026-07-16 00:47:31 ot_train.py:649 step:53K smpl:3M ep:77K epch:15.65 loss:0.022 grdn:0.243 lr:3.8e-06 updt_s:0.884 data_s:0.782 smp/s:38 mem_gb:39.65
285
+ Training: 89%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 53600/60000 [23:38:12<3:21:39, 1.89s/step]INFO 2026-07-16 00:53:12 ot_train.py:649 step:54K smpl:3M ep:77K epch:15.71 loss:0.022 grdn:0.244 lr:3.5e-06 updt_s:0.886 data_s:0.815 smp/s:38 mem_gb:39.65
286
+ Training: 90%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 53800/60000 [23:43:56<3:04:58, 1.79s/step]INFO 2026-07-16 00:58:56 ot_train.py:649 step:54K smpl:3M ep:78K epch:15.77 loss:0.022 grdn:0.258 lr:3.3e-06 updt_s:0.882 data_s:0.835 smp/s:37 mem_gb:39.65
287
+ Training: 90%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 54000/60000 [23:49:43<2:45:20, 1.65s/step]INFO 2026-07-16 01:04:43 ot_train.py:649 step:54K smpl:3M ep:78K epch:15.83 loss:0.022 grdn:0.258 lr:3.1e-06 updt_s:0.907 data_s:0.823 smp/s:37 mem_gb:39.65
288
+ Training: 90%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 54200/60000 [23:55:20<2:40:00, 1.66s/step]INFO 2026-07-16 01:10:20 ot_train.py:649 step:54K smpl:3M ep:78K epch:15.89 loss:0.021 grdn:0.248 lr:2.9e-06 updt_s:0.901 data_s:0.776 smp/s:38 mem_gb:39.63
289
+ Training: 91%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 54400/60000 [24:01:14<2:34:35, 1.66s/step]INFO 2026-07-16 01:16:14 ot_train.py:649 step:54K smpl:3M ep:79K epch:15.94 loss:0.022 grdn:0.243 lr:2.7e-06 updt_s:0.900 data_s:0.866 smp/s:36 mem_gb:39.65
290
+ Training: 91%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 54600/60000 [24:07:14<3:35:52, 2.40s/step]INFO 2026-07-16 01:22:14 ot_train.py:649 step:55K smpl:3M ep:79K epch:16.00 loss:0.022 grdn:0.245 lr:2.5e-06 updt_s:0.873 data_s:0.923 smp/s:36 mem_gb:39.65
291
+ Training: 91%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 54800/60000 [24:14:35<2:28:07, 1.71s/step]INFO 2026-07-16 01:29:35 ot_train.py:649 step:55K smpl:4M ep:79K epch:16.06 loss:0.022 grdn:0.245 lr:2.4e-06 updt_s:0.897 data_s:1.302 smp/s:29 mem_gb:39.65
292
+ Training: 92%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 55000/60000 [24:21:50<2:54:07, 2.09s/step]INFO 2026-07-16 01:36:51 ot_train.py:649 step:55K smpl:4M ep:79K epch:16.12 loss:0.022 grdn:0.251 lr:2.2e-06 updt_s:0.907 data_s:1.264 smp/s:29 mem_gb:39.65
293
+ Training: 92%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 55200/60000 [24:28:00<2:22:39, 1.78s/step]INFO 2026-07-16 01:43:01 ot_train.py:649 step:55K smpl:4M ep:80K epch:16.18 loss:0.022 grdn:0.247 lr:2.0e-06 updt_s:0.874 data_s:0.971 smp/s:35 mem_gb:39.65
294
+ Training: 92%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 55400/60000 [24:33:46<2:07:59, 1.67s/step]INFO 2026-07-16 01:48:47 ot_train.py:649 step:55K smpl:4M ep:80K epch:16.24 loss:0.022 grdn:0.244 lr:1.9e-06 updt_s:0.873 data_s:0.851 smp/s:37 mem_gb:39.65
295
+ Training: 93%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž| 55600/60000 [24:39:34<2:02:12, 1.67s/step]INFO 2026-07-16 01:54:34 ot_train.py:649 step:56K smpl:4M ep:80K epch:16.30 loss:0.021 grdn:0.245 lr:1.7e-06 updt_s:0.879 data_s:0.854 smp/s:37 mem_gb:39.65
296
+ Training: 93%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž| 55800/60000 [24:45:09<1:58:28, 1.69s/step]INFO 2026-07-16 02:00:09 ot_train.py:649 step:56K smpl:4M ep:81K epch:16.35 loss:0.022 grdn:0.247 lr:1.6e-06 updt_s:0.869 data_s:0.802 smp/s:38 mem_gb:39.65
297
+ Training: 93%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž| 56000/60000 [24:50:43<1:49:40, 1.65s/step]INFO 2026-07-16 02:05:43 ot_train.py:649 step:56K smpl:4M ep:81K epch:16.41 loss:0.022 grdn:0.269 lr:1.4e-06 updt_s:0.878 data_s:0.786 smp/s:38 mem_gb:39.65
298
+ Training: 94%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž| 56200/60000 [24:56:18<1:46:32, 1.68s/step]INFO 2026-07-16 02:11:18 ot_train.py:649 step:56K smpl:4M ep:81K epch:16.47 loss:0.021 grdn:0.257 lr:1.3e-06 updt_s:0.883 data_s:0.786 smp/s:38 mem_gb:39.65
299
+ Training: 94%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 56400/60000 [25:01:59<1:39:39, 1.66s/step]INFO 2026-07-16 02:16:59 ot_train.py:649 step:56K smpl:4M ep:81K epch:16.53 loss:0.022 grdn:0.242 lr:1.2e-06 updt_s:0.947 data_s:0.753 smp/s:38 mem_gb:39.65
300
+ Training: 94%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 56600/60000 [25:07:49<1:34:02, 1.66s/step]INFO 2026-07-16 02:22:49 ot_train.py:649 step:57K smpl:4M ep:82K epch:16.59 loss:0.021 grdn:0.251 lr:1.0e-06 updt_s:0.910 data_s:0.837 smp/s:37 mem_gb:39.63
301
+ Training: 95%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–| 56800/60000 [25:13:19<1:26:40, 1.63s/step]INFO 2026-07-16 02:28:20 ot_train.py:649 step:57K smpl:4M ep:82K epch:16.65 loss:0.021 grdn:0.235 lr:9.2e-07 updt_s:0.903 data_s:0.743 smp/s:39 mem_gb:39.65
302
+ Training: 95%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ| 57000/60000 [25:18:44<1:19:03, 1.58s/step]INFO 2026-07-16 02:33:44 ot_train.py:649 step:57K smpl:4M ep:82K epch:16.71 loss:0.021 grdn:0.237 lr:8.1e-07 updt_s:0.916 data_s:0.704 smp/s:39 mem_gb:39.65
303
+ Training: 95%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ| 57200/60000 [25:24:10<1:17:39, 1.66s/step]INFO 2026-07-16 02:39:10 ot_train.py:649 step:57K smpl:4M ep:83K epch:16.76 loss:0.021 grdn:0.243 lr:7.1e-07 updt_s:0.901 data_s:0.722 smp/s:39 mem_gb:39.65
304
+ Training: 96%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ| 57400/60000 [25:29:44<1:12:50, 1.68s/step]INFO 2026-07-16 02:44:45 ot_train.py:649 step:57K smpl:4M ep:83K epch:16.82 loss:0.021 grdn:0.253 lr:6.2e-07 updt_s:0.887 data_s:0.781 smp/s:38 mem_gb:39.65
305
+ Training: 96%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ| 57600/60000 [25:35:11<1:04:41, 1.62s/step]INFO 2026-07-16 02:50:11 ot_train.py:649 step:58K smpl:4M ep:83K epch:16.88 loss:0.021 grdn:0.244 lr:5.3e-07 updt_s:0.955 data_s:0.674 smp/s:39 mem_gb:39.65
306
+ Training: 96%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹| 57800/60000 [25:40:45<58:23, 1.59s/step] INFO 2026-07-16 02:55:45 ot_train.py:649 step:58K smpl:4M ep:84K epch:16.94 loss:0.021 grdn:0.239 lr:4.5e-07 updt_s:0.937 data_s:0.728 smp/s:38 mem_gb:39.65
307
+ Training: 97%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹| 58000/60000 [25:46:17<55:19, 1.66s/step]INFO 2026-07-16 03:01:17 ot_train.py:649 step:58K smpl:4M ep:84K epch:17.00 loss:0.021 grdn:0.243 lr:3.7e-07 updt_s:0.870 data_s:0.783 smp/s:39 mem_gb:39.65
308
+ Training: 97%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹| 58200/60000 [25:51:50<45:44, 1.52s/step]INFO 2026-07-16 03:06:50 ot_train.py:649 step:58K smpl:4M ep:84K epch:17.06 loss:0.021 grdn:0.232 lr:3.1e-07 updt_s:0.918 data_s:0.743 smp/s:39 mem_gb:39.65
309
+ Training: 97%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹| 58400/60000 [25:57:12<43:25, 1.63s/step]INFO 2026-07-16 03:12:12 ot_train.py:649 step:58K smpl:4M ep:84K epch:17.12 loss:0.021 grdn:0.243 lr:2.4e-07 updt_s:0.985 data_s:0.621 smp/s:40 mem_gb:39.65
310
+ Training: 98%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š| 58600/60000 [26:02:43<36:31, 1.57s/step]INFO 2026-07-16 03:17:44 ot_train.py:649 step:59K smpl:4M ep:85K epch:17.17 loss:0.022 grdn:0.244 lr:1.9e-07 updt_s:0.937 data_s:0.713 smp/s:39 mem_gb:39.65
311
+ Training: 98%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š| 58800/60000 [26:08:21<34:48, 1.74s/step]INFO 2026-07-16 03:23:22 ot_train.py:649 step:59K smpl:4M ep:85K epch:17.23 loss:0.021 grdn:0.230 lr:1.4e-07 updt_s:0.911 data_s:0.774 smp/s:38 mem_gb:39.63
312
+ Training: 98%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š| 59000/60000 [26:13:57<29:22, 1.76s/step]INFO 2026-07-16 03:28:57 ot_train.py:649 step:59K smpl:4M ep:85K epch:17.29 loss:0.022 grdn:0.253 lr:1.0e-07 updt_s:0.871 data_s:0.801 smp/s:38 mem_gb:39.65
313
+ Training: 99%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š| 59200/60000 [26:19:32<21:55, 1.64s/step]INFO 2026-07-16 03:34:32 ot_train.py:649 step:59K smpl:4M ep:86K epch:17.35 loss:0.021 grdn:0.248 lr:6.9e-08 updt_s:0.882 data_s:0.789 smp/s:38 mem_gb:39.65
314
+ Training: 99%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰| 59400/60000 [26:25:06<16:52, 1.69s/step]INFO 2026-07-16 03:40:07 ot_train.py:649 step:59K smpl:4M ep:86K epch:17.41 loss:0.022 grdn:0.244 lr:4.2e-08 updt_s:0.914 data_s:0.752 smp/s:38 mem_gb:39.65
315
+ Training: 99%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰| 59600/60000 [26:30:42<11:19, 1.70s/step]INFO 2026-07-16 03:45:42 ot_train.py:649 step:60K smpl:4M ep:86K epch:17.47 loss:0.021 grdn:0.239 lr:2.1e-08 updt_s:0.875 data_s:0.798 smp/s:38 mem_gb:39.65
316
+ Training: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰| 59800/60000 [26:36:16<05:38, 1.69s/step]INFO 2026-07-16 03:51:17 ot_train.py:649 step:60K smpl:4M ep:86K epch:17.53 loss:0.021 grdn:0.235 lr:7.9e-09 updt_s:0.888 data_s:0.780 smp/s:38 mem_gb:39.65
317
+ Training: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 60000/60000 [26:41:52<00:00, 1.65s/step]INFO 2026-07-16 03:56:52 ot_train.py:649 step:60K smpl:4M ep:87K epch:17.59 loss:0.021 grdn:0.238 lr:1.1e-09 updt_s:0.878 data_s:0.795 smp/s:38 mem_gb:39.65
318
+ INFO 2026-07-16 03:56:52 ot_train.py:694 Checkpoint policy after step 60000
319
+ INFO 2026-07-16 03:57:25 ain_yaml.py:918 Saved config, action-space, prompt, and phase provenance in /home/ext_minje/groot_insight/Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K and /home/ext_minje/groot_insight/Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/checkpoints/060000
320
+ Training: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 60000/60000 [26:42:25<00:00, 1.60s/step]
321
+ INFO 2026-07-16 03:57:25 ot_train.py:780 End of training
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff