Add files using upload-large-folder tool
Browse files- .gitattributes +1 -0
- Abs_Joint/Baseline/action_space_manifest.json +124 -0
- Abs_Joint/Baseline/checkpoints/060000/action_space_manifest.json +124 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/action_space_manifest.json +124 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/config.json +107 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/model.safetensors +3 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor.json +23 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors +3 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor.json +78 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors +3 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/prompt_manifest.json +40 -0
- Abs_Joint/Baseline/checkpoints/060000/pretrained_model/train_config.json +258 -0
- Abs_Joint/Baseline/checkpoints/060000/prompt_manifest.json +40 -0
- Abs_Joint/Baseline/checkpoints/060000/resolved_config.yaml +24 -0
- Abs_Joint/Baseline/checkpoints/060000/source_config.yaml +63 -0
- Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_param_groups.json +577 -0
- Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_state.safetensors +3 -0
- Abs_Joint/Baseline/checkpoints/060000/training_state/rng_state.safetensors +3 -0
- Abs_Joint/Baseline/checkpoints/060000/training_state/scheduler_state.json +18 -0
- Abs_Joint/Baseline/checkpoints/060000/training_state/training_step.json +5 -0
- Abs_Joint/Baseline/prompt_manifest.json +40 -0
- Abs_Joint/Baseline/resolved_config.yaml +24 -0
- Abs_Joint/Baseline/source_config.yaml +63 -0
- Abs_Joint/Baseline/wandb/debug-internal.log +0 -0
- Abs_Joint/Baseline/wandb/debug.log +45 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/config.yaml +300 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/output.log +320 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/requirements.txt +117 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-metadata.json +64 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-summary.json +1 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-core.log +16 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log +0 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log +45 -0
- Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/run-a3ogbmg4.wandb +3 -0
.gitattributes
CHANGED
|
@@ -43,3 +43,4 @@ Abs_6D/Baseline/wandb/run-20260714_161240-1jmjb68j/run-1jmjb68j.wandb filter=lfs
|
|
| 43 |
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/run-pebh9ode.wandb filter=lfs diff=lfs merge=lfs -text
|
| 44 |
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_10K/wandb/run-20260715_011448-fbqzmp5m/run-fbqzmp5m.wandb filter=lfs diff=lfs merge=lfs -text
|
| 45 |
Abs_6D/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260715_102129-typqzcxc/run-typqzcxc.wandb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 43 |
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_6K/wandb/run-20260715_011448-pebh9ode/run-pebh9ode.wandb filter=lfs diff=lfs merge=lfs -text
|
| 44 |
Abs_6D/RKD_TimewarpVAE/LAPstyle_linear_10K/wandb/run-20260715_011448-fbqzmp5m/run-fbqzmp5m.wandb filter=lfs diff=lfs merge=lfs -text
|
| 45 |
Abs_6D/RKD_TimewarpVAE/RSCLstyle_cosine_60K/wandb/run-20260715_102129-typqzcxc/run-typqzcxc.wandb filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/run-a3ogbmg4.wandb filter=lfs diff=lfs merge=lfs -text
|
Abs_Joint/Baseline/action_space_manifest.json
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 2,
|
| 3 |
+
"action_space": "joint_abs",
|
| 4 |
+
"action_views": [
|
| 5 |
+
"joint_abs"
|
| 6 |
+
],
|
| 7 |
+
"source_feature": "action.joint_abs",
|
| 8 |
+
"canonical_feature": "action",
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"shape": [
|
| 11 |
+
8
|
| 12 |
+
],
|
| 13 |
+
"names": [
|
| 14 |
+
"j1",
|
| 15 |
+
"j2",
|
| 16 |
+
"j3",
|
| 17 |
+
"j4",
|
| 18 |
+
"j5",
|
| 19 |
+
"j6",
|
| 20 |
+
"j7",
|
| 21 |
+
"gripper"
|
| 22 |
+
],
|
| 23 |
+
"dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
|
| 24 |
+
"dataset_revision": "v3.0",
|
| 25 |
+
"dataset_total_frames": 218367,
|
| 26 |
+
"dataset_total_episodes": 4930,
|
| 27 |
+
"dataset_fps": 10,
|
| 28 |
+
"stats_sha256": "4a35bfed6a90ddd1401509423c3b13ffc9adbcc4ffe461e1be341859505c2cc3",
|
| 29 |
+
"stats": {
|
| 30 |
+
"min": [
|
| 31 |
+
-2.8872478008270264,
|
| 32 |
+
-1.7627999782562256,
|
| 33 |
+
-2.8033182621002197,
|
| 34 |
+
-3.0717999935150146,
|
| 35 |
+
-2.887302875518799,
|
| 36 |
+
-0.012209548614919186,
|
| 37 |
+
-2.9217312335968018,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"max": [
|
| 41 |
+
2.880279302597046,
|
| 42 |
+
1.5605382919311523,
|
| 43 |
+
2.884284496307373,
|
| 44 |
+
-0.7739977240562439,
|
| 45 |
+
2.8875293731689453,
|
| 46 |
+
3.7483208179473877,
|
| 47 |
+
2.9222261905670166,
|
| 48 |
+
0.03999999910593033
|
| 49 |
+
],
|
| 50 |
+
"mean": [
|
| 51 |
+
0.0019617623927398265,
|
| 52 |
+
-0.6096056340962691,
|
| 53 |
+
-0.08146021469498298,
|
| 54 |
+
-1.9318156007993963,
|
| 55 |
+
0.13481075339339976,
|
| 56 |
+
2.1488292862501455,
|
| 57 |
+
0.20664679119827345,
|
| 58 |
+
0.021167300266524632
|
| 59 |
+
],
|
| 60 |
+
"std": [
|
| 61 |
+
0.4184733225263563,
|
| 62 |
+
0.43452186167386714,
|
| 63 |
+
0.34657299197762675,
|
| 64 |
+
0.31065299277285635,
|
| 65 |
+
0.5804164362256183,
|
| 66 |
+
0.560995779575591,
|
| 67 |
+
1.4846336752340794,
|
| 68 |
+
0.01996590431050817
|
| 69 |
+
],
|
| 70 |
+
"count": [
|
| 71 |
+
218367
|
| 72 |
+
],
|
| 73 |
+
"q01": [
|
| 74 |
+
-0.21136459407246144,
|
| 75 |
+
-1.1820493696591017,
|
| 76 |
+
-0.23617874574281436,
|
| 77 |
+
-2.3513137651342415,
|
| 78 |
+
-0.13724740781131373,
|
| 79 |
+
1.8803846119895578,
|
| 80 |
+
-1.1764070030629525,
|
| 81 |
+
-1.000000013351432e-10
|
| 82 |
+
],
|
| 83 |
+
"q10": [
|
| 84 |
+
-0.17717514597274497,
|
| 85 |
+
-0.9819092294296998,
|
| 86 |
+
-0.21149320673037963,
|
| 87 |
+
-2.1958533677694447,
|
| 88 |
+
-0.0927211042478635,
|
| 89 |
+
1.9384504318973095,
|
| 90 |
+
-0.9679293888972026,
|
| 91 |
+
-1.000000013351432e-10
|
| 92 |
+
],
|
| 93 |
+
"q50": [
|
| 94 |
+
0.004311903393904223,
|
| 95 |
+
-0.5611143444409522,
|
| 96 |
+
-0.09504186980105549,
|
| 97 |
+
-1.8999666287578785,
|
| 98 |
+
0.155166053430988,
|
| 99 |
+
2.113795242440475,
|
| 100 |
+
0.2503060374532495,
|
| 101 |
+
0.020772594934870346
|
| 102 |
+
],
|
| 103 |
+
"q90": [
|
| 104 |
+
0.16607161790312464,
|
| 105 |
+
-0.4243375581079769,
|
| 106 |
+
0.08872114509119819,
|
| 107 |
+
-1.7717546326449354,
|
| 108 |
+
0.302495674015553,
|
| 109 |
+
2.4791201268598524,
|
| 110 |
+
1.2829547587923422,
|
| 111 |
+
0.039998362213352394
|
| 112 |
+
],
|
| 113 |
+
"q99": [
|
| 114 |
+
0.19894254444240483,
|
| 115 |
+
-0.3908672692345395,
|
| 116 |
+
0.13510704087246417,
|
| 117 |
+
-1.7149038786707966,
|
| 118 |
+
0.33183211224695647,
|
| 119 |
+
2.6996600881852637,
|
| 120 |
+
1.4893457903653573,
|
| 121 |
+
0.03999983541667277
|
| 122 |
+
]
|
| 123 |
+
}
|
| 124 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/action_space_manifest.json
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 2,
|
| 3 |
+
"action_space": "joint_abs",
|
| 4 |
+
"action_views": [
|
| 5 |
+
"joint_abs"
|
| 6 |
+
],
|
| 7 |
+
"source_feature": "action.joint_abs",
|
| 8 |
+
"canonical_feature": "action",
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"shape": [
|
| 11 |
+
8
|
| 12 |
+
],
|
| 13 |
+
"names": [
|
| 14 |
+
"j1",
|
| 15 |
+
"j2",
|
| 16 |
+
"j3",
|
| 17 |
+
"j4",
|
| 18 |
+
"j5",
|
| 19 |
+
"j6",
|
| 20 |
+
"j7",
|
| 21 |
+
"gripper"
|
| 22 |
+
],
|
| 23 |
+
"dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
|
| 24 |
+
"dataset_revision": "v3.0",
|
| 25 |
+
"dataset_total_frames": 218367,
|
| 26 |
+
"dataset_total_episodes": 4930,
|
| 27 |
+
"dataset_fps": 10,
|
| 28 |
+
"stats_sha256": "4a35bfed6a90ddd1401509423c3b13ffc9adbcc4ffe461e1be341859505c2cc3",
|
| 29 |
+
"stats": {
|
| 30 |
+
"min": [
|
| 31 |
+
-2.8872478008270264,
|
| 32 |
+
-1.7627999782562256,
|
| 33 |
+
-2.8033182621002197,
|
| 34 |
+
-3.0717999935150146,
|
| 35 |
+
-2.887302875518799,
|
| 36 |
+
-0.012209548614919186,
|
| 37 |
+
-2.9217312335968018,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"max": [
|
| 41 |
+
2.880279302597046,
|
| 42 |
+
1.5605382919311523,
|
| 43 |
+
2.884284496307373,
|
| 44 |
+
-0.7739977240562439,
|
| 45 |
+
2.8875293731689453,
|
| 46 |
+
3.7483208179473877,
|
| 47 |
+
2.9222261905670166,
|
| 48 |
+
0.03999999910593033
|
| 49 |
+
],
|
| 50 |
+
"mean": [
|
| 51 |
+
0.0019617623927398265,
|
| 52 |
+
-0.6096056340962691,
|
| 53 |
+
-0.08146021469498298,
|
| 54 |
+
-1.9318156007993963,
|
| 55 |
+
0.13481075339339976,
|
| 56 |
+
2.1488292862501455,
|
| 57 |
+
0.20664679119827345,
|
| 58 |
+
0.021167300266524632
|
| 59 |
+
],
|
| 60 |
+
"std": [
|
| 61 |
+
0.4184733225263563,
|
| 62 |
+
0.43452186167386714,
|
| 63 |
+
0.34657299197762675,
|
| 64 |
+
0.31065299277285635,
|
| 65 |
+
0.5804164362256183,
|
| 66 |
+
0.560995779575591,
|
| 67 |
+
1.4846336752340794,
|
| 68 |
+
0.01996590431050817
|
| 69 |
+
],
|
| 70 |
+
"count": [
|
| 71 |
+
218367
|
| 72 |
+
],
|
| 73 |
+
"q01": [
|
| 74 |
+
-0.21136459407246144,
|
| 75 |
+
-1.1820493696591017,
|
| 76 |
+
-0.23617874574281436,
|
| 77 |
+
-2.3513137651342415,
|
| 78 |
+
-0.13724740781131373,
|
| 79 |
+
1.8803846119895578,
|
| 80 |
+
-1.1764070030629525,
|
| 81 |
+
-1.000000013351432e-10
|
| 82 |
+
],
|
| 83 |
+
"q10": [
|
| 84 |
+
-0.17717514597274497,
|
| 85 |
+
-0.9819092294296998,
|
| 86 |
+
-0.21149320673037963,
|
| 87 |
+
-2.1958533677694447,
|
| 88 |
+
-0.0927211042478635,
|
| 89 |
+
1.9384504318973095,
|
| 90 |
+
-0.9679293888972026,
|
| 91 |
+
-1.000000013351432e-10
|
| 92 |
+
],
|
| 93 |
+
"q50": [
|
| 94 |
+
0.004311903393904223,
|
| 95 |
+
-0.5611143444409522,
|
| 96 |
+
-0.09504186980105549,
|
| 97 |
+
-1.8999666287578785,
|
| 98 |
+
0.155166053430988,
|
| 99 |
+
2.113795242440475,
|
| 100 |
+
0.2503060374532495,
|
| 101 |
+
0.020772594934870346
|
| 102 |
+
],
|
| 103 |
+
"q90": [
|
| 104 |
+
0.16607161790312464,
|
| 105 |
+
-0.4243375581079769,
|
| 106 |
+
0.08872114509119819,
|
| 107 |
+
-1.7717546326449354,
|
| 108 |
+
0.302495674015553,
|
| 109 |
+
2.4791201268598524,
|
| 110 |
+
1.2829547587923422,
|
| 111 |
+
0.039998362213352394
|
| 112 |
+
],
|
| 113 |
+
"q99": [
|
| 114 |
+
0.19894254444240483,
|
| 115 |
+
-0.3908672692345395,
|
| 116 |
+
0.13510704087246417,
|
| 117 |
+
-1.7149038786707966,
|
| 118 |
+
0.33183211224695647,
|
| 119 |
+
2.6996600881852637,
|
| 120 |
+
1.4893457903653573,
|
| 121 |
+
0.03999983541667277
|
| 122 |
+
]
|
| 123 |
+
}
|
| 124 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/action_space_manifest.json
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 2,
|
| 3 |
+
"action_space": "joint_abs",
|
| 4 |
+
"action_views": [
|
| 5 |
+
"joint_abs"
|
| 6 |
+
],
|
| 7 |
+
"source_feature": "action.joint_abs",
|
| 8 |
+
"canonical_feature": "action",
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"shape": [
|
| 11 |
+
8
|
| 12 |
+
],
|
| 13 |
+
"names": [
|
| 14 |
+
"j1",
|
| 15 |
+
"j2",
|
| 16 |
+
"j3",
|
| 17 |
+
"j4",
|
| 18 |
+
"j5",
|
| 19 |
+
"j6",
|
| 20 |
+
"j7",
|
| 21 |
+
"gripper"
|
| 22 |
+
],
|
| 23 |
+
"dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
|
| 24 |
+
"dataset_revision": "v3.0",
|
| 25 |
+
"dataset_total_frames": 218367,
|
| 26 |
+
"dataset_total_episodes": 4930,
|
| 27 |
+
"dataset_fps": 10,
|
| 28 |
+
"stats_sha256": "4a35bfed6a90ddd1401509423c3b13ffc9adbcc4ffe461e1be341859505c2cc3",
|
| 29 |
+
"stats": {
|
| 30 |
+
"min": [
|
| 31 |
+
-2.8872478008270264,
|
| 32 |
+
-1.7627999782562256,
|
| 33 |
+
-2.8033182621002197,
|
| 34 |
+
-3.0717999935150146,
|
| 35 |
+
-2.887302875518799,
|
| 36 |
+
-0.012209548614919186,
|
| 37 |
+
-2.9217312335968018,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"max": [
|
| 41 |
+
2.880279302597046,
|
| 42 |
+
1.5605382919311523,
|
| 43 |
+
2.884284496307373,
|
| 44 |
+
-0.7739977240562439,
|
| 45 |
+
2.8875293731689453,
|
| 46 |
+
3.7483208179473877,
|
| 47 |
+
2.9222261905670166,
|
| 48 |
+
0.03999999910593033
|
| 49 |
+
],
|
| 50 |
+
"mean": [
|
| 51 |
+
0.0019617623927398265,
|
| 52 |
+
-0.6096056340962691,
|
| 53 |
+
-0.08146021469498298,
|
| 54 |
+
-1.9318156007993963,
|
| 55 |
+
0.13481075339339976,
|
| 56 |
+
2.1488292862501455,
|
| 57 |
+
0.20664679119827345,
|
| 58 |
+
0.021167300266524632
|
| 59 |
+
],
|
| 60 |
+
"std": [
|
| 61 |
+
0.4184733225263563,
|
| 62 |
+
0.43452186167386714,
|
| 63 |
+
0.34657299197762675,
|
| 64 |
+
0.31065299277285635,
|
| 65 |
+
0.5804164362256183,
|
| 66 |
+
0.560995779575591,
|
| 67 |
+
1.4846336752340794,
|
| 68 |
+
0.01996590431050817
|
| 69 |
+
],
|
| 70 |
+
"count": [
|
| 71 |
+
218367
|
| 72 |
+
],
|
| 73 |
+
"q01": [
|
| 74 |
+
-0.21136459407246144,
|
| 75 |
+
-1.1820493696591017,
|
| 76 |
+
-0.23617874574281436,
|
| 77 |
+
-2.3513137651342415,
|
| 78 |
+
-0.13724740781131373,
|
| 79 |
+
1.8803846119895578,
|
| 80 |
+
-1.1764070030629525,
|
| 81 |
+
-1.000000013351432e-10
|
| 82 |
+
],
|
| 83 |
+
"q10": [
|
| 84 |
+
-0.17717514597274497,
|
| 85 |
+
-0.9819092294296998,
|
| 86 |
+
-0.21149320673037963,
|
| 87 |
+
-2.1958533677694447,
|
| 88 |
+
-0.0927211042478635,
|
| 89 |
+
1.9384504318973095,
|
| 90 |
+
-0.9679293888972026,
|
| 91 |
+
-1.000000013351432e-10
|
| 92 |
+
],
|
| 93 |
+
"q50": [
|
| 94 |
+
0.004311903393904223,
|
| 95 |
+
-0.5611143444409522,
|
| 96 |
+
-0.09504186980105549,
|
| 97 |
+
-1.8999666287578785,
|
| 98 |
+
0.155166053430988,
|
| 99 |
+
2.113795242440475,
|
| 100 |
+
0.2503060374532495,
|
| 101 |
+
0.020772594934870346
|
| 102 |
+
],
|
| 103 |
+
"q90": [
|
| 104 |
+
0.16607161790312464,
|
| 105 |
+
-0.4243375581079769,
|
| 106 |
+
0.08872114509119819,
|
| 107 |
+
-1.7717546326449354,
|
| 108 |
+
0.302495674015553,
|
| 109 |
+
2.4791201268598524,
|
| 110 |
+
1.2829547587923422,
|
| 111 |
+
0.039998362213352394
|
| 112 |
+
],
|
| 113 |
+
"q99": [
|
| 114 |
+
0.19894254444240483,
|
| 115 |
+
-0.3908672692345395,
|
| 116 |
+
0.13510704087246417,
|
| 117 |
+
-1.7149038786707966,
|
| 118 |
+
0.33183211224695647,
|
| 119 |
+
2.6996600881852637,
|
| 120 |
+
1.4893457903653573,
|
| 121 |
+
0.03999983541667277
|
| 122 |
+
]
|
| 123 |
+
}
|
| 124 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/config.json
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"type": "groot",
|
| 3 |
+
"n_obs_steps": 1,
|
| 4 |
+
"input_features": {
|
| 5 |
+
"observation.state": {
|
| 6 |
+
"type": "STATE",
|
| 7 |
+
"shape": [
|
| 8 |
+
16
|
| 9 |
+
]
|
| 10 |
+
},
|
| 11 |
+
"observation.images.wrist": {
|
| 12 |
+
"type": "VISUAL",
|
| 13 |
+
"shape": [
|
| 14 |
+
3,
|
| 15 |
+
224,
|
| 16 |
+
224
|
| 17 |
+
]
|
| 18 |
+
},
|
| 19 |
+
"observation.images.right_shoulder": {
|
| 20 |
+
"type": "VISUAL",
|
| 21 |
+
"shape": [
|
| 22 |
+
3,
|
| 23 |
+
224,
|
| 24 |
+
224
|
| 25 |
+
]
|
| 26 |
+
},
|
| 27 |
+
"observation.images.guide": {
|
| 28 |
+
"type": "VISUAL",
|
| 29 |
+
"shape": [
|
| 30 |
+
3,
|
| 31 |
+
224,
|
| 32 |
+
224
|
| 33 |
+
]
|
| 34 |
+
}
|
| 35 |
+
},
|
| 36 |
+
"output_features": {
|
| 37 |
+
"action": {
|
| 38 |
+
"type": "ACTION",
|
| 39 |
+
"shape": [
|
| 40 |
+
8
|
| 41 |
+
]
|
| 42 |
+
}
|
| 43 |
+
},
|
| 44 |
+
"device": "cuda",
|
| 45 |
+
"use_amp": false,
|
| 46 |
+
"use_peft": false,
|
| 47 |
+
"push_to_hub": false,
|
| 48 |
+
"repo_id": null,
|
| 49 |
+
"private": null,
|
| 50 |
+
"tags": null,
|
| 51 |
+
"license": null,
|
| 52 |
+
"pretrained_path": null,
|
| 53 |
+
"pretrained_revision": null,
|
| 54 |
+
"chunk_size": 16,
|
| 55 |
+
"n_action_steps": 16,
|
| 56 |
+
"max_state_dim": 132,
|
| 57 |
+
"max_action_dim": 132,
|
| 58 |
+
"normalization_mapping": {
|
| 59 |
+
"VISUAL": "IDENTITY",
|
| 60 |
+
"STATE": "IDENTITY",
|
| 61 |
+
"ACTION": "IDENTITY"
|
| 62 |
+
},
|
| 63 |
+
"base_model_path": "nvidia/GR00T-N1.7-3B",
|
| 64 |
+
"action_decode_transform": null,
|
| 65 |
+
"embodiment_tag": "new_embodiment",
|
| 66 |
+
"tune_llm": false,
|
| 67 |
+
"tune_visual": false,
|
| 68 |
+
"tune_projector": true,
|
| 69 |
+
"tune_diffusion_model": true,
|
| 70 |
+
"tune_vlln": true,
|
| 71 |
+
"tune_top_llm_layers": 0,
|
| 72 |
+
"num_inference_timesteps": null,
|
| 73 |
+
"rtc_ramp_rate": null,
|
| 74 |
+
"use_flash_attention": false,
|
| 75 |
+
"use_relative_actions": false,
|
| 76 |
+
"relative_exclude_joints": [],
|
| 77 |
+
"optimizer_lr": 0.0001,
|
| 78 |
+
"optimizer_betas": [
|
| 79 |
+
0.9,
|
| 80 |
+
0.999
|
| 81 |
+
],
|
| 82 |
+
"optimizer_eps": 1e-08,
|
| 83 |
+
"optimizer_weight_decay": 1e-05,
|
| 84 |
+
"warmup_ratio": 0.05,
|
| 85 |
+
"use_bf16": true,
|
| 86 |
+
"model_params_fp32": true,
|
| 87 |
+
"image_size": [
|
| 88 |
+
256,
|
| 89 |
+
256
|
| 90 |
+
],
|
| 91 |
+
"tokenizer_assets_repo": null,
|
| 92 |
+
"lora_rank": 0,
|
| 93 |
+
"lora_alpha": 16,
|
| 94 |
+
"lora_dropout": 0.1,
|
| 95 |
+
"lora_full_model": false,
|
| 96 |
+
"video_backend": "decord",
|
| 97 |
+
"balance_dataset_weights": true,
|
| 98 |
+
"balance_trajectory_weights": true,
|
| 99 |
+
"dataset_paths": null,
|
| 100 |
+
"output_dir": "./tmp/gr00t",
|
| 101 |
+
"save_steps": 1000,
|
| 102 |
+
"max_steps": 10000,
|
| 103 |
+
"batch_size": 32,
|
| 104 |
+
"dataloader_num_workers": 8,
|
| 105 |
+
"report_to": "wandb",
|
| 106 |
+
"resume": false
|
| 107 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cf3e0351d7c4f730a00c9e511dc72d7be818363889f68430b3251e8ad1a99747
|
| 3 |
+
size 12576215296
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "policy_postprocessor",
|
| 3 |
+
"steps": [
|
| 4 |
+
{
|
| 5 |
+
"registry_name": "groot_action_unpack_unnormalize_v2",
|
| 6 |
+
"config": {
|
| 7 |
+
"env_action_dim": 8,
|
| 8 |
+
"normalize_min_max": true,
|
| 9 |
+
"clip_normalized_action": true,
|
| 10 |
+
"libero_gripper_action": false,
|
| 11 |
+
"libero_gripper_binarize": true
|
| 12 |
+
},
|
| 13 |
+
"state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors"
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"registry_name": "device_processor",
|
| 17 |
+
"config": {
|
| 18 |
+
"device": "cpu",
|
| 19 |
+
"float_dtype": null
|
| 20 |
+
}
|
| 21 |
+
}
|
| 22 |
+
]
|
| 23 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f21dbfaa54b712cd34c86e2f401159550987ceb71c54934d3e7fa600b69b15f8
|
| 3 |
+
size 5608
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor.json
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "policy_preprocessor",
|
| 3 |
+
"steps": [
|
| 4 |
+
{
|
| 5 |
+
"registry_name": "rename_observations_processor",
|
| 6 |
+
"config": {
|
| 7 |
+
"rename_map": {}
|
| 8 |
+
}
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"registry_name": "to_batch_processor",
|
| 12 |
+
"config": {}
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"registry_name": "groot_n1_7_pack_inputs_v1",
|
| 16 |
+
"config": {
|
| 17 |
+
"state_horizon": 1,
|
| 18 |
+
"action_horizon": 16,
|
| 19 |
+
"valid_action_horizon": 16,
|
| 20 |
+
"video_horizon": null,
|
| 21 |
+
"max_state_dim": 132,
|
| 22 |
+
"max_action_dim": 132,
|
| 23 |
+
"language_key": "task",
|
| 24 |
+
"formalize_language": true,
|
| 25 |
+
"embodiment_tag": "new_embodiment",
|
| 26 |
+
"embodiment_mapping": {
|
| 27 |
+
"oxe_droid_relative_eef_relative_joint": 24,
|
| 28 |
+
"xdof_relative_eef_relative_joint": 27,
|
| 29 |
+
"xdof_relative_eef_relative_joint_subtask": 27,
|
| 30 |
+
"real_g1_relative_eef_relative_joints": 25,
|
| 31 |
+
"real_r1_pro_sharpa_relative_eef": 26,
|
| 32 |
+
"real_r1_pro_sharpa_relative_eef_human": 26,
|
| 33 |
+
"real_r1_pro_sharpa_relative_eef_maxinsights": 26,
|
| 34 |
+
"real_r1_pro_sharpa_relative_eef_mecka": 26,
|
| 35 |
+
"unitree_g1_full_body_with_waist_height_nav_cmd": 25,
|
| 36 |
+
"simpler_env_google": 0,
|
| 37 |
+
"simpler_env_widowx": 1,
|
| 38 |
+
"libero_sim": 2,
|
| 39 |
+
"new_embodiment": 10
|
| 40 |
+
},
|
| 41 |
+
"normalize_min_max": true,
|
| 42 |
+
"state_dropout_prob": 0.0,
|
| 43 |
+
"clip_outliers": true,
|
| 44 |
+
"use_percentiles": false,
|
| 45 |
+
"video_modality_keys": null,
|
| 46 |
+
"raw_stats": null,
|
| 47 |
+
"modality_config": null
|
| 48 |
+
},
|
| 49 |
+
"state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"registry_name": "groot_n1_7_vlm_encode_v1",
|
| 53 |
+
"config": {
|
| 54 |
+
"model_name": "nvidia/Cosmos-Reason2-2B",
|
| 55 |
+
"image_crop_size": [
|
| 56 |
+
230,
|
| 57 |
+
230
|
| 58 |
+
],
|
| 59 |
+
"image_target_size": [
|
| 60 |
+
256,
|
| 61 |
+
256
|
| 62 |
+
],
|
| 63 |
+
"shortest_image_edge": null,
|
| 64 |
+
"crop_fraction": null,
|
| 65 |
+
"use_albumentations": false,
|
| 66 |
+
"letter_box_transform": false,
|
| 67 |
+
"device": "cuda"
|
| 68 |
+
}
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"registry_name": "device_processor",
|
| 72 |
+
"config": {
|
| 73 |
+
"device": "cuda",
|
| 74 |
+
"float_dtype": null
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
]
|
| 78 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f21dbfaa54b712cd34c86e2f401159550987ceb71c54934d3e7fa600b69b15f8
|
| 3 |
+
size 5608
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/prompt_manifest.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"prompt_set": "detailed",
|
| 4 |
+
"source_file": "task_prompts.json",
|
| 5 |
+
"source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
|
| 6 |
+
"mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
|
| 7 |
+
"dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
|
| 8 |
+
"dataset_revision": "v3.0",
|
| 9 |
+
"task_count": 13,
|
| 10 |
+
"task_codes": [
|
| 11 |
+
"5f",
|
| 12 |
+
"5g",
|
| 13 |
+
"1ext",
|
| 14 |
+
"3b",
|
| 15 |
+
"5b",
|
| 16 |
+
"3a",
|
| 17 |
+
"3c",
|
| 18 |
+
"5h",
|
| 19 |
+
"5d",
|
| 20 |
+
"3d",
|
| 21 |
+
"5e",
|
| 22 |
+
"5a",
|
| 23 |
+
"5c"
|
| 24 |
+
],
|
| 25 |
+
"prompts": {
|
| 26 |
+
"5f": "Close the bottle in clockwise direction.",
|
| 27 |
+
"5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
|
| 28 |
+
"1ext": "Find the arrow guide and open the indicated drawer.",
|
| 29 |
+
"3b": "Open the door, rotate counter-clockwise and push.",
|
| 30 |
+
"5b": "Open the bottle in counter-clockwise direction.",
|
| 31 |
+
"3a": "Open the door, rotate clockwise and push.",
|
| 32 |
+
"3c": "Open the door, rotate clockwise and pull.",
|
| 33 |
+
"5h": "Close the bottle in counter-clockwise direction.",
|
| 34 |
+
"5d": "Close the bottle in clockwise direction.",
|
| 35 |
+
"3d": "Open the door, rotate counter-clockwise and pull.",
|
| 36 |
+
"5e": "Close the bottle in counter-clockwise direction.",
|
| 37 |
+
"5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
|
| 38 |
+
"5c": "Open the bottle in clockwise direction."
|
| 39 |
+
}
|
| 40 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/pretrained_model/train_config.json
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": {
|
| 3 |
+
"repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
|
| 4 |
+
"root": "/home/seonho/INSIGHTfixposV4_filtered_multispace_v2",
|
| 5 |
+
"episodes": null,
|
| 6 |
+
"image_transforms": {
|
| 7 |
+
"enable": false,
|
| 8 |
+
"max_num_transforms": 3,
|
| 9 |
+
"random_order": false,
|
| 10 |
+
"tfs": {
|
| 11 |
+
"brightness": {
|
| 12 |
+
"weight": 1.0,
|
| 13 |
+
"type": "ColorJitter",
|
| 14 |
+
"kwargs": {
|
| 15 |
+
"brightness": [
|
| 16 |
+
0.8,
|
| 17 |
+
1.2
|
| 18 |
+
]
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
"contrast": {
|
| 22 |
+
"weight": 1.0,
|
| 23 |
+
"type": "ColorJitter",
|
| 24 |
+
"kwargs": {
|
| 25 |
+
"contrast": [
|
| 26 |
+
0.8,
|
| 27 |
+
1.2
|
| 28 |
+
]
|
| 29 |
+
}
|
| 30 |
+
},
|
| 31 |
+
"saturation": {
|
| 32 |
+
"weight": 1.0,
|
| 33 |
+
"type": "ColorJitter",
|
| 34 |
+
"kwargs": {
|
| 35 |
+
"saturation": [
|
| 36 |
+
0.5,
|
| 37 |
+
1.5
|
| 38 |
+
]
|
| 39 |
+
}
|
| 40 |
+
},
|
| 41 |
+
"hue": {
|
| 42 |
+
"weight": 1.0,
|
| 43 |
+
"type": "ColorJitter",
|
| 44 |
+
"kwargs": {
|
| 45 |
+
"hue": [
|
| 46 |
+
-0.05,
|
| 47 |
+
0.05
|
| 48 |
+
]
|
| 49 |
+
}
|
| 50 |
+
},
|
| 51 |
+
"sharpness": {
|
| 52 |
+
"weight": 1.0,
|
| 53 |
+
"type": "SharpnessJitter",
|
| 54 |
+
"kwargs": {
|
| 55 |
+
"sharpness": [
|
| 56 |
+
0.5,
|
| 57 |
+
1.5
|
| 58 |
+
]
|
| 59 |
+
}
|
| 60 |
+
},
|
| 61 |
+
"affine": {
|
| 62 |
+
"weight": 1.0,
|
| 63 |
+
"type": "RandomAffine",
|
| 64 |
+
"kwargs": {
|
| 65 |
+
"degrees": [
|
| 66 |
+
-5.0,
|
| 67 |
+
5.0
|
| 68 |
+
],
|
| 69 |
+
"translate": [
|
| 70 |
+
0.05,
|
| 71 |
+
0.05
|
| 72 |
+
]
|
| 73 |
+
}
|
| 74 |
+
}
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
"revision": null,
|
| 78 |
+
"use_imagenet_stats": true,
|
| 79 |
+
"video_backend": "torchcodec",
|
| 80 |
+
"return_uint8": false,
|
| 81 |
+
"depth_output_unit": "mm",
|
| 82 |
+
"streaming": false,
|
| 83 |
+
"eval_split": 0.0
|
| 84 |
+
},
|
| 85 |
+
"env": null,
|
| 86 |
+
"policy": {
|
| 87 |
+
"type": "groot",
|
| 88 |
+
"n_obs_steps": 1,
|
| 89 |
+
"input_features": {
|
| 90 |
+
"observation.state": {
|
| 91 |
+
"type": "STATE",
|
| 92 |
+
"shape": [
|
| 93 |
+
16
|
| 94 |
+
]
|
| 95 |
+
},
|
| 96 |
+
"observation.images.wrist": {
|
| 97 |
+
"type": "VISUAL",
|
| 98 |
+
"shape": [
|
| 99 |
+
3,
|
| 100 |
+
224,
|
| 101 |
+
224
|
| 102 |
+
]
|
| 103 |
+
},
|
| 104 |
+
"observation.images.right_shoulder": {
|
| 105 |
+
"type": "VISUAL",
|
| 106 |
+
"shape": [
|
| 107 |
+
3,
|
| 108 |
+
224,
|
| 109 |
+
224
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"observation.images.guide": {
|
| 113 |
+
"type": "VISUAL",
|
| 114 |
+
"shape": [
|
| 115 |
+
3,
|
| 116 |
+
224,
|
| 117 |
+
224
|
| 118 |
+
]
|
| 119 |
+
}
|
| 120 |
+
},
|
| 121 |
+
"output_features": {
|
| 122 |
+
"action": {
|
| 123 |
+
"type": "ACTION",
|
| 124 |
+
"shape": [
|
| 125 |
+
8
|
| 126 |
+
]
|
| 127 |
+
}
|
| 128 |
+
},
|
| 129 |
+
"device": "cuda",
|
| 130 |
+
"use_amp": false,
|
| 131 |
+
"use_peft": false,
|
| 132 |
+
"push_to_hub": false,
|
| 133 |
+
"repo_id": null,
|
| 134 |
+
"private": null,
|
| 135 |
+
"tags": null,
|
| 136 |
+
"license": null,
|
| 137 |
+
"pretrained_path": null,
|
| 138 |
+
"pretrained_revision": null,
|
| 139 |
+
"chunk_size": 16,
|
| 140 |
+
"n_action_steps": 16,
|
| 141 |
+
"max_state_dim": 132,
|
| 142 |
+
"max_action_dim": 132,
|
| 143 |
+
"normalization_mapping": {
|
| 144 |
+
"VISUAL": "IDENTITY",
|
| 145 |
+
"STATE": "IDENTITY",
|
| 146 |
+
"ACTION": "IDENTITY"
|
| 147 |
+
},
|
| 148 |
+
"base_model_path": "nvidia/GR00T-N1.7-3B",
|
| 149 |
+
"action_decode_transform": null,
|
| 150 |
+
"embodiment_tag": "new_embodiment",
|
| 151 |
+
"tune_llm": false,
|
| 152 |
+
"tune_visual": false,
|
| 153 |
+
"tune_projector": true,
|
| 154 |
+
"tune_diffusion_model": true,
|
| 155 |
+
"tune_vlln": true,
|
| 156 |
+
"tune_top_llm_layers": 0,
|
| 157 |
+
"num_inference_timesteps": null,
|
| 158 |
+
"rtc_ramp_rate": null,
|
| 159 |
+
"use_flash_attention": false,
|
| 160 |
+
"use_relative_actions": false,
|
| 161 |
+
"relative_exclude_joints": [],
|
| 162 |
+
"optimizer_lr": 0.0001,
|
| 163 |
+
"optimizer_betas": [
|
| 164 |
+
0.9,
|
| 165 |
+
0.999
|
| 166 |
+
],
|
| 167 |
+
"optimizer_eps": 1e-08,
|
| 168 |
+
"optimizer_weight_decay": 1e-05,
|
| 169 |
+
"warmup_ratio": 0.05,
|
| 170 |
+
"use_bf16": true,
|
| 171 |
+
"model_params_fp32": true,
|
| 172 |
+
"image_size": [
|
| 173 |
+
256,
|
| 174 |
+
256
|
| 175 |
+
],
|
| 176 |
+
"tokenizer_assets_repo": null,
|
| 177 |
+
"lora_rank": 0,
|
| 178 |
+
"lora_alpha": 16,
|
| 179 |
+
"lora_dropout": 0.1,
|
| 180 |
+
"lora_full_model": false,
|
| 181 |
+
"video_backend": "decord",
|
| 182 |
+
"balance_dataset_weights": true,
|
| 183 |
+
"balance_trajectory_weights": true,
|
| 184 |
+
"dataset_paths": null,
|
| 185 |
+
"output_dir": "./tmp/gr00t",
|
| 186 |
+
"save_steps": 1000,
|
| 187 |
+
"max_steps": 10000,
|
| 188 |
+
"batch_size": 32,
|
| 189 |
+
"dataloader_num_workers": 8,
|
| 190 |
+
"report_to": "wandb",
|
| 191 |
+
"resume": false
|
| 192 |
+
},
|
| 193 |
+
"reward_model": null,
|
| 194 |
+
"output_dir": "/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline",
|
| 195 |
+
"job_name": "INSIGHT_RD_baseline",
|
| 196 |
+
"resume": false,
|
| 197 |
+
"seed": 42,
|
| 198 |
+
"cudnn_deterministic": false,
|
| 199 |
+
"num_workers": 4,
|
| 200 |
+
"batch_size": 128,
|
| 201 |
+
"prefetch_factor": 4,
|
| 202 |
+
"persistent_workers": true,
|
| 203 |
+
"steps": 60000,
|
| 204 |
+
"env_eval_freq": 20000,
|
| 205 |
+
"log_freq": 200,
|
| 206 |
+
"eval_steps": 0,
|
| 207 |
+
"max_eval_samples": 0,
|
| 208 |
+
"tolerance_s": 0.0001,
|
| 209 |
+
"save_checkpoint": true,
|
| 210 |
+
"save_freq": 20000,
|
| 211 |
+
"use_policy_training_preset": true,
|
| 212 |
+
"optimizer": {
|
| 213 |
+
"type": "adamw",
|
| 214 |
+
"lr": 0.0001,
|
| 215 |
+
"weight_decay": 1e-05,
|
| 216 |
+
"grad_clip_norm": 1.0,
|
| 217 |
+
"betas": [
|
| 218 |
+
0.9,
|
| 219 |
+
0.999
|
| 220 |
+
],
|
| 221 |
+
"eps": 1e-08
|
| 222 |
+
},
|
| 223 |
+
"scheduler": {
|
| 224 |
+
"type": "diffuser",
|
| 225 |
+
"num_warmup_steps": 500,
|
| 226 |
+
"name": "cosine"
|
| 227 |
+
},
|
| 228 |
+
"eval": {
|
| 229 |
+
"n_episodes": 50,
|
| 230 |
+
"batch_size": 44,
|
| 231 |
+
"use_async_envs": true,
|
| 232 |
+
"recording": false,
|
| 233 |
+
"recording_repo_id": null,
|
| 234 |
+
"recording_private": false
|
| 235 |
+
},
|
| 236 |
+
"wandb": {
|
| 237 |
+
"enable": true,
|
| 238 |
+
"disable_artifact": true,
|
| 239 |
+
"project": "lerobot",
|
| 240 |
+
"entity": null,
|
| 241 |
+
"notes": null,
|
| 242 |
+
"run_id": "a3ogbmg4",
|
| 243 |
+
"mode": null,
|
| 244 |
+
"add_tags": true
|
| 245 |
+
},
|
| 246 |
+
"peft": null,
|
| 247 |
+
"job": {
|
| 248 |
+
"target": null,
|
| 249 |
+
"image": "huggingface/lerobot-gpu:latest",
|
| 250 |
+
"timeout": "2d",
|
| 251 |
+
"detach": false,
|
| 252 |
+
"tags": []
|
| 253 |
+
},
|
| 254 |
+
"save_checkpoint_to_hub": false,
|
| 255 |
+
"sample_weighting": null,
|
| 256 |
+
"rename_map": {},
|
| 257 |
+
"checkpoint_path": null
|
| 258 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/prompt_manifest.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"prompt_set": "detailed",
|
| 4 |
+
"source_file": "task_prompts.json",
|
| 5 |
+
"source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
|
| 6 |
+
"mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
|
| 7 |
+
"dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
|
| 8 |
+
"dataset_revision": "v3.0",
|
| 9 |
+
"task_count": 13,
|
| 10 |
+
"task_codes": [
|
| 11 |
+
"5f",
|
| 12 |
+
"5g",
|
| 13 |
+
"1ext",
|
| 14 |
+
"3b",
|
| 15 |
+
"5b",
|
| 16 |
+
"3a",
|
| 17 |
+
"3c",
|
| 18 |
+
"5h",
|
| 19 |
+
"5d",
|
| 20 |
+
"3d",
|
| 21 |
+
"5e",
|
| 22 |
+
"5a",
|
| 23 |
+
"5c"
|
| 24 |
+
],
|
| 25 |
+
"prompts": {
|
| 26 |
+
"5f": "Close the bottle in clockwise direction.",
|
| 27 |
+
"5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
|
| 28 |
+
"1ext": "Find the arrow guide and open the indicated drawer.",
|
| 29 |
+
"3b": "Open the door, rotate counter-clockwise and push.",
|
| 30 |
+
"5b": "Open the bottle in counter-clockwise direction.",
|
| 31 |
+
"3a": "Open the door, rotate clockwise and push.",
|
| 32 |
+
"3c": "Open the door, rotate clockwise and pull.",
|
| 33 |
+
"5h": "Close the bottle in counter-clockwise direction.",
|
| 34 |
+
"5d": "Close the bottle in clockwise direction.",
|
| 35 |
+
"3d": "Open the door, rotate counter-clockwise and pull.",
|
| 36 |
+
"5e": "Close the bottle in counter-clockwise direction.",
|
| 37 |
+
"5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
|
| 38 |
+
"5c": "Open the bottle in clockwise direction."
|
| 39 |
+
}
|
| 40 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/resolved_config.yaml
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset:
|
| 2 |
+
repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
|
| 3 |
+
root: /home/seonho/INSIGHTfixposV4_filtered_multispace_v2
|
| 4 |
+
policy:
|
| 5 |
+
type: groot
|
| 6 |
+
device: cuda
|
| 7 |
+
chunk_size: 16
|
| 8 |
+
n_action_steps: 16
|
| 9 |
+
push_to_hub: false
|
| 10 |
+
tune_llm: false
|
| 11 |
+
tune_visual: false
|
| 12 |
+
tune_top_llm_layers: 0
|
| 13 |
+
tune_projector: true
|
| 14 |
+
tune_diffusion_model: true
|
| 15 |
+
tune_vlln: true
|
| 16 |
+
seed: 42
|
| 17 |
+
batch_size: 128
|
| 18 |
+
steps: 60000
|
| 19 |
+
log_freq: 200
|
| 20 |
+
output_dir: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
|
| 21 |
+
job_name: INSIGHT_RD_baseline
|
| 22 |
+
wandb:
|
| 23 |
+
enable: true
|
| 24 |
+
disable_artifact: true
|
Abs_Joint/Baseline/checkpoints/060000/source_config.yaml
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# INSIGHT GR00T N1.7 training config
|
| 2 |
+
# INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
|
| 3 |
+
insight:
|
| 4 |
+
# Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
|
| 5 |
+
action_space: joint_abs
|
| 6 |
+
# Options: coarse | detailed
|
| 7 |
+
prompt_set: detailed
|
| 8 |
+
save_steps: ["60K"]
|
| 9 |
+
|
| 10 |
+
dataset:
|
| 11 |
+
repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
|
| 12 |
+
root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
|
| 13 |
+
|
| 14 |
+
policy:
|
| 15 |
+
type: groot
|
| 16 |
+
device: cuda
|
| 17 |
+
chunk_size: 16
|
| 18 |
+
n_action_steps: 16
|
| 19 |
+
push_to_hub: false
|
| 20 |
+
# Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
|
| 21 |
+
# VLM
|
| 22 |
+
tune_llm: false
|
| 23 |
+
tune_visual: false
|
| 24 |
+
tune_top_llm_layers: 0
|
| 25 |
+
# Action Expert
|
| 26 |
+
tune_projector: true
|
| 27 |
+
tune_diffusion_model: true
|
| 28 |
+
tune_vlln: true
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
seed: 42
|
| 32 |
+
batch_size: 128
|
| 33 |
+
steps: 60000
|
| 34 |
+
log_freq: 200
|
| 35 |
+
output_dir: "${HOME}/groot_insight/train_ckpt/Abs_Joint/Baseline"
|
| 36 |
+
job_name: INSIGHT_RD_baseline
|
| 37 |
+
|
| 38 |
+
wandb:
|
| 39 |
+
enable: true
|
| 40 |
+
disable_artifact: true
|
| 41 |
+
|
| 42 |
+
# Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
|
| 43 |
+
#
|
| 44 |
+
# dataset:
|
| 45 |
+
# eval_split: 0.0
|
| 46 |
+
# image_transforms:
|
| 47 |
+
# enable: false # default; set true only for train-image augmentation.
|
| 48 |
+
#
|
| 49 |
+
# policy:
|
| 50 |
+
# base_model_path: nvidia/GR00T-N1.7-3B
|
| 51 |
+
# embodiment_tag: new_embodiment
|
| 52 |
+
# use_relative_actions: false
|
| 53 |
+
# use_bf16: true
|
| 54 |
+
# repo_id: Whalswp/INSIGHT # unused while push_to_hub is false
|
| 55 |
+
#
|
| 56 |
+
# save_checkpoint: true
|
| 57 |
+
# use_policy_training_preset: true
|
| 58 |
+
# eval_steps: 0
|
| 59 |
+
# env_eval_freq: 0
|
| 60 |
+
#
|
| 61 |
+
# insight.prompt_set options:
|
| 62 |
+
# coarse: High-level task text that requires following the visual guide.
|
| 63 |
+
# detailed: Direction- and interaction-specific text, including rotation and push/pull details.
|
Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_param_groups.json
ADDED
|
@@ -0,0 +1,577 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"lr": 0.0,
|
| 4 |
+
"betas": [
|
| 5 |
+
0.9,
|
| 6 |
+
0.999
|
| 7 |
+
],
|
| 8 |
+
"eps": 1e-08,
|
| 9 |
+
"weight_decay": 1e-05,
|
| 10 |
+
"amsgrad": false,
|
| 11 |
+
"maximize": false,
|
| 12 |
+
"foreach": null,
|
| 13 |
+
"capturable": false,
|
| 14 |
+
"differentiable": false,
|
| 15 |
+
"fused": null,
|
| 16 |
+
"decoupled_weight_decay": true,
|
| 17 |
+
"initial_lr": 0.0001,
|
| 18 |
+
"params": [
|
| 19 |
+
0,
|
| 20 |
+
1,
|
| 21 |
+
2,
|
| 22 |
+
3,
|
| 23 |
+
4,
|
| 24 |
+
5,
|
| 25 |
+
6,
|
| 26 |
+
7,
|
| 27 |
+
8,
|
| 28 |
+
9,
|
| 29 |
+
10,
|
| 30 |
+
11,
|
| 31 |
+
12,
|
| 32 |
+
13,
|
| 33 |
+
14,
|
| 34 |
+
15,
|
| 35 |
+
16,
|
| 36 |
+
17,
|
| 37 |
+
18,
|
| 38 |
+
19,
|
| 39 |
+
20,
|
| 40 |
+
21,
|
| 41 |
+
22,
|
| 42 |
+
23,
|
| 43 |
+
24,
|
| 44 |
+
25,
|
| 45 |
+
26,
|
| 46 |
+
27,
|
| 47 |
+
28,
|
| 48 |
+
29,
|
| 49 |
+
30,
|
| 50 |
+
31,
|
| 51 |
+
32,
|
| 52 |
+
33,
|
| 53 |
+
34,
|
| 54 |
+
35,
|
| 55 |
+
36,
|
| 56 |
+
37,
|
| 57 |
+
38,
|
| 58 |
+
39,
|
| 59 |
+
40,
|
| 60 |
+
41,
|
| 61 |
+
42,
|
| 62 |
+
43,
|
| 63 |
+
44,
|
| 64 |
+
45,
|
| 65 |
+
46,
|
| 66 |
+
47,
|
| 67 |
+
48,
|
| 68 |
+
49,
|
| 69 |
+
50,
|
| 70 |
+
51,
|
| 71 |
+
52,
|
| 72 |
+
53,
|
| 73 |
+
54,
|
| 74 |
+
55,
|
| 75 |
+
56,
|
| 76 |
+
57,
|
| 77 |
+
58,
|
| 78 |
+
59,
|
| 79 |
+
60,
|
| 80 |
+
61,
|
| 81 |
+
62,
|
| 82 |
+
63,
|
| 83 |
+
64,
|
| 84 |
+
65,
|
| 85 |
+
66,
|
| 86 |
+
67,
|
| 87 |
+
68,
|
| 88 |
+
69,
|
| 89 |
+
70,
|
| 90 |
+
71,
|
| 91 |
+
72,
|
| 92 |
+
73,
|
| 93 |
+
74,
|
| 94 |
+
75,
|
| 95 |
+
76,
|
| 96 |
+
77,
|
| 97 |
+
78,
|
| 98 |
+
79,
|
| 99 |
+
80,
|
| 100 |
+
81,
|
| 101 |
+
82,
|
| 102 |
+
83,
|
| 103 |
+
84,
|
| 104 |
+
85,
|
| 105 |
+
86,
|
| 106 |
+
87,
|
| 107 |
+
88,
|
| 108 |
+
89,
|
| 109 |
+
90,
|
| 110 |
+
91,
|
| 111 |
+
92,
|
| 112 |
+
93,
|
| 113 |
+
94,
|
| 114 |
+
95,
|
| 115 |
+
96,
|
| 116 |
+
97,
|
| 117 |
+
98,
|
| 118 |
+
99,
|
| 119 |
+
100,
|
| 120 |
+
101,
|
| 121 |
+
102,
|
| 122 |
+
103,
|
| 123 |
+
104,
|
| 124 |
+
105,
|
| 125 |
+
106,
|
| 126 |
+
107,
|
| 127 |
+
108,
|
| 128 |
+
109,
|
| 129 |
+
110,
|
| 130 |
+
111,
|
| 131 |
+
112,
|
| 132 |
+
113,
|
| 133 |
+
114,
|
| 134 |
+
115,
|
| 135 |
+
116,
|
| 136 |
+
117,
|
| 137 |
+
118,
|
| 138 |
+
119,
|
| 139 |
+
120,
|
| 140 |
+
121,
|
| 141 |
+
122,
|
| 142 |
+
123,
|
| 143 |
+
124,
|
| 144 |
+
125,
|
| 145 |
+
126,
|
| 146 |
+
127,
|
| 147 |
+
128,
|
| 148 |
+
129,
|
| 149 |
+
130,
|
| 150 |
+
131,
|
| 151 |
+
132,
|
| 152 |
+
133,
|
| 153 |
+
134,
|
| 154 |
+
135,
|
| 155 |
+
136,
|
| 156 |
+
137,
|
| 157 |
+
138,
|
| 158 |
+
139,
|
| 159 |
+
140,
|
| 160 |
+
141,
|
| 161 |
+
142,
|
| 162 |
+
143,
|
| 163 |
+
144,
|
| 164 |
+
145,
|
| 165 |
+
146,
|
| 166 |
+
147,
|
| 167 |
+
148,
|
| 168 |
+
149,
|
| 169 |
+
150,
|
| 170 |
+
151,
|
| 171 |
+
152,
|
| 172 |
+
153,
|
| 173 |
+
154,
|
| 174 |
+
155,
|
| 175 |
+
156,
|
| 176 |
+
157,
|
| 177 |
+
158,
|
| 178 |
+
159,
|
| 179 |
+
160,
|
| 180 |
+
161,
|
| 181 |
+
162,
|
| 182 |
+
163,
|
| 183 |
+
164,
|
| 184 |
+
165,
|
| 185 |
+
166,
|
| 186 |
+
167,
|
| 187 |
+
168,
|
| 188 |
+
169,
|
| 189 |
+
170,
|
| 190 |
+
171,
|
| 191 |
+
172,
|
| 192 |
+
173,
|
| 193 |
+
174,
|
| 194 |
+
175,
|
| 195 |
+
176,
|
| 196 |
+
177,
|
| 197 |
+
178,
|
| 198 |
+
179,
|
| 199 |
+
180,
|
| 200 |
+
181,
|
| 201 |
+
182,
|
| 202 |
+
183,
|
| 203 |
+
184,
|
| 204 |
+
185,
|
| 205 |
+
186,
|
| 206 |
+
187,
|
| 207 |
+
188,
|
| 208 |
+
189,
|
| 209 |
+
190,
|
| 210 |
+
191,
|
| 211 |
+
192,
|
| 212 |
+
193,
|
| 213 |
+
194,
|
| 214 |
+
195,
|
| 215 |
+
196,
|
| 216 |
+
197,
|
| 217 |
+
198,
|
| 218 |
+
199,
|
| 219 |
+
200,
|
| 220 |
+
201,
|
| 221 |
+
202,
|
| 222 |
+
203,
|
| 223 |
+
204,
|
| 224 |
+
205,
|
| 225 |
+
206,
|
| 226 |
+
207,
|
| 227 |
+
208,
|
| 228 |
+
209,
|
| 229 |
+
210,
|
| 230 |
+
211,
|
| 231 |
+
212,
|
| 232 |
+
213,
|
| 233 |
+
214,
|
| 234 |
+
215,
|
| 235 |
+
216,
|
| 236 |
+
217,
|
| 237 |
+
218,
|
| 238 |
+
219,
|
| 239 |
+
220,
|
| 240 |
+
221,
|
| 241 |
+
222,
|
| 242 |
+
223,
|
| 243 |
+
224,
|
| 244 |
+
225,
|
| 245 |
+
226,
|
| 246 |
+
227,
|
| 247 |
+
228,
|
| 248 |
+
229,
|
| 249 |
+
230,
|
| 250 |
+
231,
|
| 251 |
+
232,
|
| 252 |
+
233,
|
| 253 |
+
234,
|
| 254 |
+
235,
|
| 255 |
+
236,
|
| 256 |
+
237,
|
| 257 |
+
238,
|
| 258 |
+
239,
|
| 259 |
+
240,
|
| 260 |
+
241,
|
| 261 |
+
242,
|
| 262 |
+
243,
|
| 263 |
+
244,
|
| 264 |
+
245,
|
| 265 |
+
246,
|
| 266 |
+
247,
|
| 267 |
+
248,
|
| 268 |
+
249,
|
| 269 |
+
250,
|
| 270 |
+
251,
|
| 271 |
+
252,
|
| 272 |
+
253,
|
| 273 |
+
254,
|
| 274 |
+
255,
|
| 275 |
+
256,
|
| 276 |
+
257,
|
| 277 |
+
258,
|
| 278 |
+
259,
|
| 279 |
+
260,
|
| 280 |
+
261,
|
| 281 |
+
262,
|
| 282 |
+
263,
|
| 283 |
+
264,
|
| 284 |
+
265,
|
| 285 |
+
266
|
| 286 |
+
]
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"weight_decay": 0.0,
|
| 290 |
+
"lr": 0.0,
|
| 291 |
+
"betas": [
|
| 292 |
+
0.9,
|
| 293 |
+
0.999
|
| 294 |
+
],
|
| 295 |
+
"eps": 1e-08,
|
| 296 |
+
"amsgrad": false,
|
| 297 |
+
"maximize": false,
|
| 298 |
+
"foreach": null,
|
| 299 |
+
"capturable": false,
|
| 300 |
+
"differentiable": false,
|
| 301 |
+
"fused": null,
|
| 302 |
+
"decoupled_weight_decay": true,
|
| 303 |
+
"initial_lr": 0.0001,
|
| 304 |
+
"params": [
|
| 305 |
+
267,
|
| 306 |
+
268,
|
| 307 |
+
269,
|
| 308 |
+
270,
|
| 309 |
+
271,
|
| 310 |
+
272,
|
| 311 |
+
273,
|
| 312 |
+
274,
|
| 313 |
+
275,
|
| 314 |
+
276,
|
| 315 |
+
277,
|
| 316 |
+
278,
|
| 317 |
+
279,
|
| 318 |
+
280,
|
| 319 |
+
281,
|
| 320 |
+
282,
|
| 321 |
+
283,
|
| 322 |
+
284,
|
| 323 |
+
285,
|
| 324 |
+
286,
|
| 325 |
+
287,
|
| 326 |
+
288,
|
| 327 |
+
289,
|
| 328 |
+
290,
|
| 329 |
+
291,
|
| 330 |
+
292,
|
| 331 |
+
293,
|
| 332 |
+
294,
|
| 333 |
+
295,
|
| 334 |
+
296,
|
| 335 |
+
297,
|
| 336 |
+
298,
|
| 337 |
+
299,
|
| 338 |
+
300,
|
| 339 |
+
301,
|
| 340 |
+
302,
|
| 341 |
+
303,
|
| 342 |
+
304,
|
| 343 |
+
305,
|
| 344 |
+
306,
|
| 345 |
+
307,
|
| 346 |
+
308,
|
| 347 |
+
309,
|
| 348 |
+
310,
|
| 349 |
+
311,
|
| 350 |
+
312,
|
| 351 |
+
313,
|
| 352 |
+
314,
|
| 353 |
+
315,
|
| 354 |
+
316,
|
| 355 |
+
317,
|
| 356 |
+
318,
|
| 357 |
+
319,
|
| 358 |
+
320,
|
| 359 |
+
321,
|
| 360 |
+
322,
|
| 361 |
+
323,
|
| 362 |
+
324,
|
| 363 |
+
325,
|
| 364 |
+
326,
|
| 365 |
+
327,
|
| 366 |
+
328,
|
| 367 |
+
329,
|
| 368 |
+
330,
|
| 369 |
+
331,
|
| 370 |
+
332,
|
| 371 |
+
333,
|
| 372 |
+
334,
|
| 373 |
+
335,
|
| 374 |
+
336,
|
| 375 |
+
337,
|
| 376 |
+
338,
|
| 377 |
+
339,
|
| 378 |
+
340,
|
| 379 |
+
341,
|
| 380 |
+
342,
|
| 381 |
+
343,
|
| 382 |
+
344,
|
| 383 |
+
345,
|
| 384 |
+
346,
|
| 385 |
+
347,
|
| 386 |
+
348,
|
| 387 |
+
349,
|
| 388 |
+
350,
|
| 389 |
+
351,
|
| 390 |
+
352,
|
| 391 |
+
353,
|
| 392 |
+
354,
|
| 393 |
+
355,
|
| 394 |
+
356,
|
| 395 |
+
357,
|
| 396 |
+
358,
|
| 397 |
+
359,
|
| 398 |
+
360,
|
| 399 |
+
361,
|
| 400 |
+
362,
|
| 401 |
+
363,
|
| 402 |
+
364,
|
| 403 |
+
365,
|
| 404 |
+
366,
|
| 405 |
+
367,
|
| 406 |
+
368,
|
| 407 |
+
369,
|
| 408 |
+
370,
|
| 409 |
+
371,
|
| 410 |
+
372,
|
| 411 |
+
373,
|
| 412 |
+
374,
|
| 413 |
+
375,
|
| 414 |
+
376,
|
| 415 |
+
377,
|
| 416 |
+
378,
|
| 417 |
+
379,
|
| 418 |
+
380,
|
| 419 |
+
381,
|
| 420 |
+
382,
|
| 421 |
+
383,
|
| 422 |
+
384,
|
| 423 |
+
385,
|
| 424 |
+
386,
|
| 425 |
+
387,
|
| 426 |
+
388,
|
| 427 |
+
389,
|
| 428 |
+
390,
|
| 429 |
+
391,
|
| 430 |
+
392,
|
| 431 |
+
393,
|
| 432 |
+
394,
|
| 433 |
+
395,
|
| 434 |
+
396,
|
| 435 |
+
397,
|
| 436 |
+
398,
|
| 437 |
+
399,
|
| 438 |
+
400,
|
| 439 |
+
401,
|
| 440 |
+
402,
|
| 441 |
+
403,
|
| 442 |
+
404,
|
| 443 |
+
405,
|
| 444 |
+
406,
|
| 445 |
+
407,
|
| 446 |
+
408,
|
| 447 |
+
409,
|
| 448 |
+
410,
|
| 449 |
+
411,
|
| 450 |
+
412,
|
| 451 |
+
413,
|
| 452 |
+
414,
|
| 453 |
+
415,
|
| 454 |
+
416,
|
| 455 |
+
417,
|
| 456 |
+
418,
|
| 457 |
+
419,
|
| 458 |
+
420,
|
| 459 |
+
421,
|
| 460 |
+
422,
|
| 461 |
+
423,
|
| 462 |
+
424,
|
| 463 |
+
425,
|
| 464 |
+
426,
|
| 465 |
+
427,
|
| 466 |
+
428,
|
| 467 |
+
429,
|
| 468 |
+
430,
|
| 469 |
+
431,
|
| 470 |
+
432,
|
| 471 |
+
433,
|
| 472 |
+
434,
|
| 473 |
+
435,
|
| 474 |
+
436,
|
| 475 |
+
437,
|
| 476 |
+
438,
|
| 477 |
+
439,
|
| 478 |
+
440,
|
| 479 |
+
441,
|
| 480 |
+
442,
|
| 481 |
+
443,
|
| 482 |
+
444,
|
| 483 |
+
445,
|
| 484 |
+
446,
|
| 485 |
+
447,
|
| 486 |
+
448,
|
| 487 |
+
449,
|
| 488 |
+
450,
|
| 489 |
+
451,
|
| 490 |
+
452,
|
| 491 |
+
453,
|
| 492 |
+
454,
|
| 493 |
+
455,
|
| 494 |
+
456,
|
| 495 |
+
457,
|
| 496 |
+
458,
|
| 497 |
+
459,
|
| 498 |
+
460,
|
| 499 |
+
461,
|
| 500 |
+
462,
|
| 501 |
+
463,
|
| 502 |
+
464,
|
| 503 |
+
465,
|
| 504 |
+
466,
|
| 505 |
+
467,
|
| 506 |
+
468,
|
| 507 |
+
469,
|
| 508 |
+
470,
|
| 509 |
+
471,
|
| 510 |
+
472,
|
| 511 |
+
473,
|
| 512 |
+
474,
|
| 513 |
+
475,
|
| 514 |
+
476,
|
| 515 |
+
477,
|
| 516 |
+
478,
|
| 517 |
+
479,
|
| 518 |
+
480,
|
| 519 |
+
481,
|
| 520 |
+
482,
|
| 521 |
+
483,
|
| 522 |
+
484,
|
| 523 |
+
485,
|
| 524 |
+
486,
|
| 525 |
+
487,
|
| 526 |
+
488,
|
| 527 |
+
489,
|
| 528 |
+
490,
|
| 529 |
+
491,
|
| 530 |
+
492,
|
| 531 |
+
493,
|
| 532 |
+
494,
|
| 533 |
+
495,
|
| 534 |
+
496,
|
| 535 |
+
497,
|
| 536 |
+
498,
|
| 537 |
+
499,
|
| 538 |
+
500,
|
| 539 |
+
501,
|
| 540 |
+
502,
|
| 541 |
+
503,
|
| 542 |
+
504,
|
| 543 |
+
505,
|
| 544 |
+
506,
|
| 545 |
+
507,
|
| 546 |
+
508,
|
| 547 |
+
509,
|
| 548 |
+
510,
|
| 549 |
+
511,
|
| 550 |
+
512,
|
| 551 |
+
513,
|
| 552 |
+
514,
|
| 553 |
+
515,
|
| 554 |
+
516,
|
| 555 |
+
517,
|
| 556 |
+
518,
|
| 557 |
+
519,
|
| 558 |
+
520,
|
| 559 |
+
521,
|
| 560 |
+
522,
|
| 561 |
+
523,
|
| 562 |
+
524,
|
| 563 |
+
525,
|
| 564 |
+
526,
|
| 565 |
+
527,
|
| 566 |
+
528,
|
| 567 |
+
529,
|
| 568 |
+
530,
|
| 569 |
+
531,
|
| 570 |
+
532,
|
| 571 |
+
533,
|
| 572 |
+
534,
|
| 573 |
+
535,
|
| 574 |
+
536
|
| 575 |
+
]
|
| 576 |
+
}
|
| 577 |
+
]
|
Abs_Joint/Baseline/checkpoints/060000/training_state/optimizer_state.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8a55b000921729c6ed71b96eead039d9c6d38263bebc1f68a0bf6af8a60609b1
|
| 3 |
+
size 12964276740
|
Abs_Joint/Baseline/checkpoints/060000/training_state/rng_state.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c4fa151c4df5f27995ad9e7874ec1263c3fe39807321efa6361dad12633080a0
|
| 3 |
+
size 15708
|
Abs_Joint/Baseline/checkpoints/060000/training_state/scheduler_state.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"base_lrs": [
|
| 3 |
+
0.0001,
|
| 4 |
+
0.0001
|
| 5 |
+
],
|
| 6 |
+
"last_epoch": 60000,
|
| 7 |
+
"_step_count": 60001,
|
| 8 |
+
"_is_initial": false,
|
| 9 |
+
"_get_lr_called_within_step": false,
|
| 10 |
+
"_last_lr": [
|
| 11 |
+
0.0,
|
| 12 |
+
0.0
|
| 13 |
+
],
|
| 14 |
+
"lr_lambdas": [
|
| 15 |
+
null,
|
| 16 |
+
null
|
| 17 |
+
]
|
| 18 |
+
}
|
Abs_Joint/Baseline/checkpoints/060000/training_state/training_step.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 60000,
|
| 3 |
+
"num_processes": 1,
|
| 4 |
+
"batch_size": 128
|
| 5 |
+
}
|
Abs_Joint/Baseline/prompt_manifest.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"prompt_set": "detailed",
|
| 4 |
+
"source_file": "task_prompts.json",
|
| 5 |
+
"source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc",
|
| 6 |
+
"mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018",
|
| 7 |
+
"dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2",
|
| 8 |
+
"dataset_revision": "v3.0",
|
| 9 |
+
"task_count": 13,
|
| 10 |
+
"task_codes": [
|
| 11 |
+
"5f",
|
| 12 |
+
"5g",
|
| 13 |
+
"1ext",
|
| 14 |
+
"3b",
|
| 15 |
+
"5b",
|
| 16 |
+
"3a",
|
| 17 |
+
"3c",
|
| 18 |
+
"5h",
|
| 19 |
+
"5d",
|
| 20 |
+
"3d",
|
| 21 |
+
"5e",
|
| 22 |
+
"5a",
|
| 23 |
+
"5c"
|
| 24 |
+
],
|
| 25 |
+
"prompts": {
|
| 26 |
+
"5f": "Close the bottle in clockwise direction.",
|
| 27 |
+
"5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.",
|
| 28 |
+
"1ext": "Find the arrow guide and open the indicated drawer.",
|
| 29 |
+
"3b": "Open the door, rotate counter-clockwise and push.",
|
| 30 |
+
"5b": "Open the bottle in counter-clockwise direction.",
|
| 31 |
+
"3a": "Open the door, rotate clockwise and push.",
|
| 32 |
+
"3c": "Open the door, rotate clockwise and pull.",
|
| 33 |
+
"5h": "Close the bottle in counter-clockwise direction.",
|
| 34 |
+
"5d": "Close the bottle in clockwise direction.",
|
| 35 |
+
"3d": "Open the door, rotate counter-clockwise and pull.",
|
| 36 |
+
"5e": "Close the bottle in counter-clockwise direction.",
|
| 37 |
+
"5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.",
|
| 38 |
+
"5c": "Open the bottle in clockwise direction."
|
| 39 |
+
}
|
| 40 |
+
}
|
Abs_Joint/Baseline/resolved_config.yaml
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset:
|
| 2 |
+
repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
|
| 3 |
+
root: /home/seonho/INSIGHTfixposV4_filtered_multispace_v2
|
| 4 |
+
policy:
|
| 5 |
+
type: groot
|
| 6 |
+
device: cuda
|
| 7 |
+
chunk_size: 16
|
| 8 |
+
n_action_steps: 16
|
| 9 |
+
push_to_hub: false
|
| 10 |
+
tune_llm: false
|
| 11 |
+
tune_visual: false
|
| 12 |
+
tune_top_llm_layers: 0
|
| 13 |
+
tune_projector: true
|
| 14 |
+
tune_diffusion_model: true
|
| 15 |
+
tune_vlln: true
|
| 16 |
+
seed: 42
|
| 17 |
+
batch_size: 128
|
| 18 |
+
steps: 60000
|
| 19 |
+
log_freq: 200
|
| 20 |
+
output_dir: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
|
| 21 |
+
job_name: INSIGHT_RD_baseline
|
| 22 |
+
wandb:
|
| 23 |
+
enable: true
|
| 24 |
+
disable_artifact: true
|
Abs_Joint/Baseline/source_config.yaml
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# INSIGHT GR00T N1.7 training config
|
| 2 |
+
# INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
|
| 3 |
+
insight:
|
| 4 |
+
# Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
|
| 5 |
+
action_space: joint_abs
|
| 6 |
+
# Options: coarse | detailed
|
| 7 |
+
prompt_set: detailed
|
| 8 |
+
save_steps: ["60K"]
|
| 9 |
+
|
| 10 |
+
dataset:
|
| 11 |
+
repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
|
| 12 |
+
root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
|
| 13 |
+
|
| 14 |
+
policy:
|
| 15 |
+
type: groot
|
| 16 |
+
device: cuda
|
| 17 |
+
chunk_size: 16
|
| 18 |
+
n_action_steps: 16
|
| 19 |
+
push_to_hub: false
|
| 20 |
+
# Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
|
| 21 |
+
# VLM
|
| 22 |
+
tune_llm: false
|
| 23 |
+
tune_visual: false
|
| 24 |
+
tune_top_llm_layers: 0
|
| 25 |
+
# Action Expert
|
| 26 |
+
tune_projector: true
|
| 27 |
+
tune_diffusion_model: true
|
| 28 |
+
tune_vlln: true
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
seed: 42
|
| 32 |
+
batch_size: 128
|
| 33 |
+
steps: 60000
|
| 34 |
+
log_freq: 200
|
| 35 |
+
output_dir: "${HOME}/groot_insight/train_ckpt/Abs_Joint/Baseline"
|
| 36 |
+
job_name: INSIGHT_RD_baseline
|
| 37 |
+
|
| 38 |
+
wandb:
|
| 39 |
+
enable: true
|
| 40 |
+
disable_artifact: true
|
| 41 |
+
|
| 42 |
+
# Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
|
| 43 |
+
#
|
| 44 |
+
# dataset:
|
| 45 |
+
# eval_split: 0.0
|
| 46 |
+
# image_transforms:
|
| 47 |
+
# enable: false # default; set true only for train-image augmentation.
|
| 48 |
+
#
|
| 49 |
+
# policy:
|
| 50 |
+
# base_model_path: nvidia/GR00T-N1.7-3B
|
| 51 |
+
# embodiment_tag: new_embodiment
|
| 52 |
+
# use_relative_actions: false
|
| 53 |
+
# use_bf16: true
|
| 54 |
+
# repo_id: Whalswp/INSIGHT # unused while push_to_hub is false
|
| 55 |
+
#
|
| 56 |
+
# save_checkpoint: true
|
| 57 |
+
# use_policy_training_preset: true
|
| 58 |
+
# eval_steps: 0
|
| 59 |
+
# env_eval_freq: 0
|
| 60 |
+
#
|
| 61 |
+
# insight.prompt_set options:
|
| 62 |
+
# coarse: High-level task text that requires following the visual guide.
|
| 63 |
+
# detailed: Direction- and interaction-specific text, including rotation and push/pull details.
|
Abs_Joint/Baseline/wandb/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
Abs_Joint/Baseline/wandb/debug.log
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
|
| 2 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Configure stats pid to 3896698
|
| 3 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log
|
| 5 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log
|
| 6 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():766] calling init triggers
|
| 7 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/seonho/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 44, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
|
| 9 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():814] starting backend
|
| 10 |
+
2026-07-17 15:48:31,110 INFO MainThread:3896698 [wandb_init.py:init():829] sending inform_init request
|
| 11 |
+
2026-07-17 15:48:31,565 INFO MainThread:3896698 [wandb_init.py:init():834] backend started and connected
|
| 12 |
+
2026-07-17 15:48:31,568 INFO MainThread:3896698 [wandb_init.py:init():904] updated telemetry
|
| 13 |
+
2026-07-17 15:48:31,579 INFO MainThread:3896698 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-17 15:48:32,131 INFO MainThread:3896698 [wandb_init.py:init():972] starting run threads in backend
|
| 15 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_console_start():2523] atexit reg
|
| 16 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-07-17 15:48:32,267 INFO MainThread:3896698 [wandb_init.py:init():1010] run started, returning control to user process
|
| 20 |
+
2026-07-20 02:13:16,832 INFO wandb-AsyncioManager-main:3896698 [service_client.py:_forward_responses():122] Reached EOF.
|
| 21 |
+
2026-07-20 02:13:16,833 INFO wandb-AsyncioManager-main:3896698 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
|
| 22 |
+
2026-07-20 02:13:16,834 ERROR wandb-AsyncioManager-main:3896698 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
|
| 23 |
+
Traceback (most recent call last):
|
| 24 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
|
| 25 |
+
await fn()
|
| 26 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
|
| 27 |
+
await asyncio_compat.race(
|
| 28 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
|
| 29 |
+
async with open_task_group(race=True) as tg:
|
| 30 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 31 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
|
| 32 |
+
await anext(self.gen)
|
| 33 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
|
| 34 |
+
await task_group._wait_all(race=race, timeout=exit_timeout)
|
| 35 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
|
| 36 |
+
raise exc
|
| 37 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
|
| 38 |
+
result = await handle.wait_async(timeout=None)
|
| 39 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 40 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
|
| 41 |
+
response = await self._handle.wait_async(timeout=timeout)
|
| 42 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 43 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
|
| 44 |
+
raise HandleAbandonedError()
|
| 45 |
+
wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/config.yaml
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.27.2
|
| 4 |
+
e:
|
| 5 |
+
pbf296fi8oqpaqrnve886ppcvwjoxaoj:
|
| 6 |
+
args:
|
| 7 |
+
- --config_path=/tmp/insight_lerobot_cfg_s1elc9y_/baseline.yaml
|
| 8 |
+
- --batch_size=128
|
| 9 |
+
codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
|
| 10 |
+
codePathLocal: my_scripts/lerobot_train_yaml.py
|
| 11 |
+
cpu_count: 32
|
| 12 |
+
cpu_count_logical: 64
|
| 13 |
+
cudaVersion: "13.2"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "3778130485248"
|
| 17 |
+
used: "3148893519872"
|
| 18 |
+
executable: /home/seonho/miniconda3/envs/lerobot060_groot/bin/python
|
| 19 |
+
git:
|
| 20 |
+
commit: a970d9f98f189faccce13cf2fb61ccf5d55921f3
|
| 21 |
+
remote: git@github.com:paragon7060/clvla.git
|
| 22 |
+
gpu: NVIDIA A100 80GB PCIe
|
| 23 |
+
gpu_count: 4
|
| 24 |
+
gpu_nvidia:
|
| 25 |
+
- architecture: Ampere
|
| 26 |
+
cudaCores: 6912
|
| 27 |
+
memoryTotal: "85899345920"
|
| 28 |
+
name: NVIDIA A100 80GB PCIe
|
| 29 |
+
uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9
|
| 30 |
+
- architecture: Ampere
|
| 31 |
+
cudaCores: 6912
|
| 32 |
+
memoryTotal: "85899345920"
|
| 33 |
+
name: NVIDIA A100 80GB PCIe
|
| 34 |
+
uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa
|
| 35 |
+
- architecture: Ampere
|
| 36 |
+
cudaCores: 6912
|
| 37 |
+
memoryTotal: "85899345920"
|
| 38 |
+
name: NVIDIA A100 80GB PCIe
|
| 39 |
+
uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac
|
| 40 |
+
- architecture: Ampere
|
| 41 |
+
cudaCores: 6912
|
| 42 |
+
memoryTotal: "85899345920"
|
| 43 |
+
name: NVIDIA A100 80GB PCIe
|
| 44 |
+
uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d
|
| 45 |
+
host: worker1
|
| 46 |
+
memory:
|
| 47 |
+
total: "1081620799488"
|
| 48 |
+
os: Linux-6.8.0-110-generic-x86_64-with-glibc2.35
|
| 49 |
+
program: /home/seonho/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
|
| 50 |
+
python: CPython 3.12.13
|
| 51 |
+
root: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
|
| 52 |
+
startedAt: "2026-07-17T06:48:30.892778Z"
|
| 53 |
+
writerId: pbf296fi8oqpaqrnve886ppcvwjoxaoj
|
| 54 |
+
m: []
|
| 55 |
+
python_version: 3.12.13
|
| 56 |
+
t:
|
| 57 |
+
"1":
|
| 58 |
+
- 1
|
| 59 |
+
- 11
|
| 60 |
+
- 41
|
| 61 |
+
- 49
|
| 62 |
+
- 51
|
| 63 |
+
- 71
|
| 64 |
+
- 83
|
| 65 |
+
- 98
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 11
|
| 69 |
+
- 41
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 71
|
| 73 |
+
- 83
|
| 74 |
+
- 98
|
| 75 |
+
"3":
|
| 76 |
+
- 13
|
| 77 |
+
- 15
|
| 78 |
+
- 16
|
| 79 |
+
- 61
|
| 80 |
+
"4": 3.12.13
|
| 81 |
+
"5": 0.27.2
|
| 82 |
+
"6": 5.5.4
|
| 83 |
+
"10":
|
| 84 |
+
- 21
|
| 85 |
+
"12": 0.27.2
|
| 86 |
+
"13": linux-x86_64
|
| 87 |
+
batch_size:
|
| 88 |
+
value: 128
|
| 89 |
+
checkpoint_path:
|
| 90 |
+
value: null
|
| 91 |
+
cudnn_deterministic:
|
| 92 |
+
value: false
|
| 93 |
+
dataset:
|
| 94 |
+
value:
|
| 95 |
+
depth_output_unit: mm
|
| 96 |
+
episodes: null
|
| 97 |
+
eval_split: 0
|
| 98 |
+
image_transforms:
|
| 99 |
+
enable: false
|
| 100 |
+
max_num_transforms: 3
|
| 101 |
+
random_order: false
|
| 102 |
+
tfs:
|
| 103 |
+
affine:
|
| 104 |
+
kwargs:
|
| 105 |
+
degrees:
|
| 106 |
+
- -5
|
| 107 |
+
- 5
|
| 108 |
+
translate:
|
| 109 |
+
- 0.05
|
| 110 |
+
- 0.05
|
| 111 |
+
type: RandomAffine
|
| 112 |
+
weight: 1
|
| 113 |
+
brightness:
|
| 114 |
+
kwargs:
|
| 115 |
+
brightness:
|
| 116 |
+
- 0.8
|
| 117 |
+
- 1.2
|
| 118 |
+
type: ColorJitter
|
| 119 |
+
weight: 1
|
| 120 |
+
contrast:
|
| 121 |
+
kwargs:
|
| 122 |
+
contrast:
|
| 123 |
+
- 0.8
|
| 124 |
+
- 1.2
|
| 125 |
+
type: ColorJitter
|
| 126 |
+
weight: 1
|
| 127 |
+
hue:
|
| 128 |
+
kwargs:
|
| 129 |
+
hue:
|
| 130 |
+
- -0.05
|
| 131 |
+
- 0.05
|
| 132 |
+
type: ColorJitter
|
| 133 |
+
weight: 1
|
| 134 |
+
saturation:
|
| 135 |
+
kwargs:
|
| 136 |
+
saturation:
|
| 137 |
+
- 0.5
|
| 138 |
+
- 1.5
|
| 139 |
+
type: ColorJitter
|
| 140 |
+
weight: 1
|
| 141 |
+
sharpness:
|
| 142 |
+
kwargs:
|
| 143 |
+
sharpness:
|
| 144 |
+
- 0.5
|
| 145 |
+
- 1.5
|
| 146 |
+
type: SharpnessJitter
|
| 147 |
+
weight: 1
|
| 148 |
+
repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
|
| 149 |
+
return_uint8: false
|
| 150 |
+
revision: null
|
| 151 |
+
root: /home/seonho/INSIGHTfixposV4_filtered_multispace_v2
|
| 152 |
+
streaming: false
|
| 153 |
+
use_imagenet_stats: true
|
| 154 |
+
video_backend: torchcodec
|
| 155 |
+
env:
|
| 156 |
+
value: null
|
| 157 |
+
env_eval_freq:
|
| 158 |
+
value: 20000
|
| 159 |
+
eval:
|
| 160 |
+
value:
|
| 161 |
+
batch_size: 44
|
| 162 |
+
n_episodes: 50
|
| 163 |
+
recording: false
|
| 164 |
+
recording_private: false
|
| 165 |
+
recording_repo_id: null
|
| 166 |
+
use_async_envs: true
|
| 167 |
+
eval_steps:
|
| 168 |
+
value: 0
|
| 169 |
+
job:
|
| 170 |
+
value:
|
| 171 |
+
detach: false
|
| 172 |
+
image: huggingface/lerobot-gpu:latest
|
| 173 |
+
tags: []
|
| 174 |
+
target: null
|
| 175 |
+
timeout: 2d
|
| 176 |
+
job_name:
|
| 177 |
+
value: INSIGHT_RD_baseline
|
| 178 |
+
log_freq:
|
| 179 |
+
value: 200
|
| 180 |
+
max_eval_samples:
|
| 181 |
+
value: 0
|
| 182 |
+
num_workers:
|
| 183 |
+
value: 4
|
| 184 |
+
optimizer:
|
| 185 |
+
value:
|
| 186 |
+
betas:
|
| 187 |
+
- 0.9
|
| 188 |
+
- 0.999
|
| 189 |
+
eps: 1e-08
|
| 190 |
+
grad_clip_norm: 1
|
| 191 |
+
lr: 0.0001
|
| 192 |
+
type: adamw
|
| 193 |
+
weight_decay: 1e-05
|
| 194 |
+
output_dir:
|
| 195 |
+
value: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
|
| 196 |
+
peft:
|
| 197 |
+
value: null
|
| 198 |
+
persistent_workers:
|
| 199 |
+
value: true
|
| 200 |
+
policy:
|
| 201 |
+
value:
|
| 202 |
+
action_decode_transform: null
|
| 203 |
+
balance_dataset_weights: true
|
| 204 |
+
balance_trajectory_weights: true
|
| 205 |
+
base_model_path: nvidia/GR00T-N1.7-3B
|
| 206 |
+
batch_size: 32
|
| 207 |
+
chunk_size: 16
|
| 208 |
+
dataloader_num_workers: 8
|
| 209 |
+
dataset_paths: null
|
| 210 |
+
device: cuda
|
| 211 |
+
embodiment_tag: new_embodiment
|
| 212 |
+
image_size:
|
| 213 |
+
- 256
|
| 214 |
+
- 256
|
| 215 |
+
license: null
|
| 216 |
+
lora_alpha: 16
|
| 217 |
+
lora_dropout: 0.1
|
| 218 |
+
lora_full_model: false
|
| 219 |
+
lora_rank: 0
|
| 220 |
+
max_action_dim: 132
|
| 221 |
+
max_state_dim: 132
|
| 222 |
+
max_steps: 10000
|
| 223 |
+
model_params_fp32: true
|
| 224 |
+
n_action_steps: 16
|
| 225 |
+
n_obs_steps: 1
|
| 226 |
+
normalization_mapping:
|
| 227 |
+
ACTION: IDENTITY
|
| 228 |
+
STATE: IDENTITY
|
| 229 |
+
VISUAL: IDENTITY
|
| 230 |
+
num_inference_timesteps: null
|
| 231 |
+
optimizer_betas:
|
| 232 |
+
- 0.9
|
| 233 |
+
- 0.999
|
| 234 |
+
optimizer_eps: 1e-08
|
| 235 |
+
optimizer_lr: 0.0001
|
| 236 |
+
optimizer_weight_decay: 1e-05
|
| 237 |
+
output_dir: ./tmp/gr00t
|
| 238 |
+
pretrained_path: null
|
| 239 |
+
pretrained_revision: null
|
| 240 |
+
private: null
|
| 241 |
+
push_to_hub: false
|
| 242 |
+
relative_exclude_joints: []
|
| 243 |
+
repo_id: null
|
| 244 |
+
report_to: wandb
|
| 245 |
+
resume: false
|
| 246 |
+
rtc_ramp_rate: null
|
| 247 |
+
save_steps: 1000
|
| 248 |
+
tags: null
|
| 249 |
+
tokenizer_assets_repo: null
|
| 250 |
+
tune_diffusion_model: true
|
| 251 |
+
tune_llm: false
|
| 252 |
+
tune_projector: true
|
| 253 |
+
tune_top_llm_layers: 0
|
| 254 |
+
tune_visual: false
|
| 255 |
+
tune_vlln: true
|
| 256 |
+
type: groot
|
| 257 |
+
use_amp: false
|
| 258 |
+
use_bf16: true
|
| 259 |
+
use_flash_attention: false
|
| 260 |
+
use_peft: false
|
| 261 |
+
use_relative_actions: false
|
| 262 |
+
video_backend: decord
|
| 263 |
+
warmup_ratio: 0.05
|
| 264 |
+
prefetch_factor:
|
| 265 |
+
value: 4
|
| 266 |
+
resume:
|
| 267 |
+
value: false
|
| 268 |
+
reward_model:
|
| 269 |
+
value: null
|
| 270 |
+
sample_weighting:
|
| 271 |
+
value: null
|
| 272 |
+
save_checkpoint:
|
| 273 |
+
value: true
|
| 274 |
+
save_checkpoint_to_hub:
|
| 275 |
+
value: false
|
| 276 |
+
save_freq:
|
| 277 |
+
value: 20000
|
| 278 |
+
scheduler:
|
| 279 |
+
value:
|
| 280 |
+
name: cosine
|
| 281 |
+
num_warmup_steps: 500
|
| 282 |
+
type: diffuser
|
| 283 |
+
seed:
|
| 284 |
+
value: 42
|
| 285 |
+
steps:
|
| 286 |
+
value: 60000
|
| 287 |
+
tolerance_s:
|
| 288 |
+
value: 0.0001
|
| 289 |
+
use_policy_training_preset:
|
| 290 |
+
value: true
|
| 291 |
+
wandb:
|
| 292 |
+
value:
|
| 293 |
+
add_tags: true
|
| 294 |
+
disable_artifact: true
|
| 295 |
+
enable: true
|
| 296 |
+
entity: null
|
| 297 |
+
mode: null
|
| 298 |
+
notes: null
|
| 299 |
+
project: lerobot
|
| 300 |
+
run_id: null
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/output.log
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
INFO 2026-07-17 15:48:32 db_utils.py:121 Logs will be synced with wandb.
|
| 2 |
+
INFO 2026-07-17 15:48:32 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/a3ogbmg4
|
| 3 |
+
INFO 2026-07-17 15:48:32 ot_train.py:298 Creating dataset
|
| 4 |
+
INFO 2026-07-17 15:48:34 ot_train.py:332 Creating policy
|
| 5 |
+
Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 6565.38it/s]
|
| 6 |
+
`torch_dtype` is deprecated! Use `dtype` instead!
|
| 7 |
+
Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 2551.60it/s]
|
| 8 |
+
INFO 2026-07-17 15:48:38 ot_train.py:405 Creating optimizer and scheduler
|
| 9 |
+
INFO 2026-07-17 15:48:38 ot_train.py:437 Output dir: /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline
|
| 10 |
+
INFO 2026-07-17 15:48:38 ot_train.py:444 cfg.steps=60000 (60K)
|
| 11 |
+
INFO 2026-07-17 15:48:38 ot_train.py:445 dataset.num_frames=218367 (218K)
|
| 12 |
+
INFO 2026-07-17 15:48:38 ot_train.py:446 dataset.num_episodes=4930
|
| 13 |
+
INFO 2026-07-17 15:48:38 ot_train.py:449 Effective batch size: 128 x 1 = 128
|
| 14 |
+
INFO 2026-07-17 15:48:38 ot_train.py:450 num_learnable_params=1620515968 (2B)
|
| 15 |
+
INFO 2026-07-17 15:48:38 ot_train.py:451 num_total_params=3144016000 (3B)
|
| 16 |
+
Training: 0%| | 0/60000 [00:00<?, ?step/s]INFO 2026-07-17 15:48:38 ot_train.py:602 Start offline training on a fixed dataset, with effective batch size: 128
|
| 17 |
+
Training: 0%| | 200/60000 [11:48<58:04:19, 3.50s/step]INFO 2026-07-17 16:00:27 ot_train.py:646 step:200 smpl:26K ep:578 epch:0.12 loss:0.753 grdn:1.151 lr:2.0e-05 updt_s:2.377 data_s:1.161 smp/s:36 mem_gb:52.70
|
| 18 |
+
Training: 1%| | 400/60000 [23:29<58:09:51, 3.51s/step]INFO 2026-07-17 16:12:08 ot_train.py:646 step:400 smpl:51K ep:1K epch:0.23 loss:0.162 grdn:1.529 lr:6.0e-05 updt_s:2.368 data_s:1.136 smp/s:37 mem_gb:52.76
|
| 19 |
+
Training: 1%| | 600/60000 [35:11<57:39:38, 3.49s/step]INFO 2026-07-17 16:23:50 ot_train.py:646 step:600 smpl:77K ep:2K epch:0.35 loss:0.123 grdn:0.938 lr:9.5e-05 updt_s:2.367 data_s:1.143 smp/s:36 mem_gb:52.76
|
| 20 |
+
Training: 1%|▏ | 800/60000 [46:53<59:16:57, 3.61s/step]INFO 2026-07-17 16:35:32 ot_train.py:646 step:800 smpl:102K ep:2K epch:0.47 loss:0.099 grdn:0.682 lr:1.0e-04 updt_s:2.370 data_s:1.137 smp/s:37 mem_gb:52.76
|
| 21 |
+
Training: 2%|▏ | 1000/60000 [58:35<58:13:04, 3.55s/step]INFO 2026-07-17 16:47:14 ot_train.py:646 step:1K smpl:128K ep:3K epch:0.59 loss:0.089 grdn:0.569 lr:1.0e-04 updt_s:2.367 data_s:1.143 smp/s:36 mem_gb:52.76
|
| 22 |
+
Training: 2%|▏ | 1200/60000 [1:10:22<56:50:49, 3.48s/step]INFO 2026-07-17 16:59:01 ot_train.py:646 step:1K smpl:154K ep:3K epch:0.70 loss:0.082 grdn:0.520 lr:1.0e-04 updt_s:2.368 data_s:1.165 smp/s:36 mem_gb:52.69
|
| 23 |
+
Training: 2%|▏ | 1400/60000 [1:22:08<57:10:38, 3.51s/step]INFO 2026-07-17 17:10:47 ot_train.py:646 step:1K smpl:179K ep:4K epch:0.82 loss:0.076 grdn:0.455 lr:1.0e-04 updt_s:2.378 data_s:1.150 smp/s:36 mem_gb:52.79
|
| 24 |
+
Training: 3%|▎ | 1600/60000 [1:33:51<56:45:55, 3.50s/step]INFO 2026-07-17 17:22:30 ot_train.py:646 step:2K smpl:205K ep:5K epch:0.94 loss:0.073 grdn:0.427 lr:1.0e-04 updt_s:2.369 data_s:1.143 smp/s:36 mem_gb:52.79
|
| 25 |
+
Training: 3%|▎ | 1800/60000 [1:45:33<57:30:15, 3.56s/step]INFO 2026-07-17 17:34:12 ot_train.py:646 step:2K smpl:230K ep:5K epch:1.06 loss:0.070 grdn:0.404 lr:1.0e-04 updt_s:2.370 data_s:1.141 smp/s:36 mem_gb:52.79
|
| 26 |
+
Training: 3%|▎ | 2000/60000 [1:57:15<56:07:26, 3.48s/step]INFO 2026-07-17 17:45:54 ot_train.py:646 step:2K smpl:256K ep:6K epch:1.17 loss:0.069 grdn:0.391 lr:1.0e-04 updt_s:2.365 data_s:1.143 smp/s:36 mem_gb:52.79
|
| 27 |
+
Training: 4%|▎ | 2200/60000 [2:08:59<56:18:20, 3.51s/step]INFO 2026-07-17 17:57:37 ot_train.py:646 step:2K smpl:282K ep:6K epch:1.29 loss:0.066 grdn:0.370 lr:1.0e-04 updt_s:2.370 data_s:1.145 smp/s:36 mem_gb:52.79
|
| 28 |
+
Training: 4%|▍ | 2400/60000 [2:20:41<56:13:18, 3.51s/step]INFO 2026-07-17 18:09:20 ot_train.py:646 step:2K smpl:307K ep:7K epch:1.41 loss:0.065 grdn:0.350 lr:1.0e-04 updt_s:2.355 data_s:1.158 smp/s:36 mem_gb:52.71
|
| 29 |
+
Training: 4%|▍ | 2600/60000 [2:32:24<56:17:48, 3.53s/step]INFO 2026-07-17 18:21:03 ot_train.py:646 step:3K smpl:333K ep:8K epch:1.52 loss:0.063 grdn:0.338 lr:1.0e-04 updt_s:2.362 data_s:1.151 smp/s:36 mem_gb:52.79
|
| 30 |
+
Training: 5%|▍ | 2800/60000 [2:44:08<56:07:25, 3.53s/step]INFO 2026-07-17 18:32:47 ot_train.py:646 step:3K smpl:358K ep:8K epch:1.64 loss:0.060 grdn:0.311 lr:1.0e-04 updt_s:2.366 data_s:1.152 smp/s:36 mem_gb:52.79
|
| 31 |
+
Training: 5%|▌ | 3000/60000 [2:55:52<56:27:17, 3.57s/step]INFO 2026-07-17 18:44:30 ot_train.py:646 step:3K smpl:384K ep:9K epch:1.76 loss:0.062 grdn:0.321 lr:1.0e-04 updt_s:2.365 data_s:1.149 smp/s:36 mem_gb:52.79
|
| 32 |
+
Training: 5%|▌ | 3200/60000 [3:07:35<55:43:02, 3.53s/step]INFO 2026-07-17 18:56:14 ot_train.py:646 step:3K smpl:410K ep:9K epch:1.88 loss:0.060 grdn:0.301 lr:1.0e-04 updt_s:2.373 data_s:1.144 smp/s:36 mem_gb:52.79
|
| 33 |
+
Training: 6%|▌ | 3400/60000 [3:19:20<54:56:01, 3.49s/step]INFO 2026-07-17 19:07:59 ot_train.py:646 step:3K smpl:435K ep:10K epch:1.99 loss:0.060 grdn:0.293 lr:9.9e-05 updt_s:2.364 data_s:1.155 smp/s:36 mem_gb:52.71
|
| 34 |
+
Training: 6%|▌ | 3600/60000 [3:31:01<55:23:52, 3.54s/step]INFO 2026-07-17 19:19:40 ot_train.py:646 step:4K smpl:461K ep:10K epch:2.11 loss:0.057 grdn:0.281 lr:9.9e-05 updt_s:2.362 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 35 |
+
Training: 6%|▋ | 3800/60000 [3:42:42<54:26:57, 3.49s/step]INFO 2026-07-17 19:31:21 ot_train.py:646 step:4K smpl:486K ep:11K epch:2.23 loss:0.057 grdn:0.283 lr:9.9e-05 updt_s:2.362 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 36 |
+
Training: 7%|▋ | 4000/60000 [3:54:22<54:28:49, 3.50s/step]INFO 2026-07-17 19:43:01 ot_train.py:646 step:4K smpl:512K ep:12K epch:2.34 loss:0.057 grdn:0.279 lr:9.9e-05 updt_s:2.359 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 37 |
+
Training: 7%|▋ | 4200/60000 [4:06:15<55:51:35, 3.60s/step]INFO 2026-07-17 19:54:54 ot_train.py:646 step:4K smpl:538K ep:12K epch:2.46 loss:0.055 grdn:0.263 lr:9.9e-05 updt_s:2.391 data_s:1.171 smp/s:36 mem_gb:52.79
|
| 38 |
+
Training: 7%|▋ | 4400/60000 [4:18:09<55:37:34, 3.60s/step]INFO 2026-07-17 20:06:48 ot_train.py:646 step:4K smpl:563K ep:13K epch:2.58 loss:0.055 grdn:0.257 lr:9.9e-05 updt_s:2.395 data_s:1.175 smp/s:36 mem_gb:52.79
|
| 39 |
+
Training: 8%|▊ | 4600/60000 [4:30:03<54:45:26, 3.56s/step]INFO 2026-07-17 20:18:42 ot_train.py:646 step:5K smpl:589K ep:13K epch:2.70 loss:0.055 grdn:0.270 lr:9.9e-05 updt_s:2.385 data_s:1.182 smp/s:36 mem_gb:52.71
|
| 40 |
+
Training: 8%|▊ | 4800/60000 [4:41:51<54:21:08, 3.54s/step]INFO 2026-07-17 20:30:30 ot_train.py:646 step:5K smpl:614K ep:14K epch:2.81 loss:0.055 grdn:0.258 lr:9.9e-05 updt_s:2.381 data_s:1.157 smp/s:36 mem_gb:52.79
|
| 41 |
+
Training: 8%|▊ | 5000/60000 [4:53:40<53:22:02, 3.49s/step]INFO 2026-07-17 20:42:19 ot_train.py:646 step:5K smpl:640K ep:14K epch:2.93 loss:0.053 grdn:0.252 lr:9.9e-05 updt_s:2.383 data_s:1.160 smp/s:36 mem_gb:52.79
|
| 42 |
+
Training: 9%|▊ | 5200/60000 [5:05:22<53:53:15, 3.54s/step]INFO 2026-07-17 20:54:01 ot_train.py:646 step:5K smpl:666K ep:15K epch:3.05 loss:0.053 grdn:0.248 lr:9.9e-05 updt_s:2.364 data_s:1.144 smp/s:36 mem_gb:52.79
|
| 43 |
+
Training: 9%|▉ | 5400/60000 [5:17:06<53:23:36, 3.52s/step]INFO 2026-07-17 21:05:45 ot_train.py:646 step:5K smpl:691K ep:16K epch:3.17 loss:0.052 grdn:0.240 lr:9.8e-05 updt_s:2.371 data_s:1.147 smp/s:36 mem_gb:52.79
|
| 44 |
+
Training: 9%|▉ | 5600/60000 [5:28:48<52:51:47, 3.50s/step]INFO 2026-07-17 21:17:27 ot_train.py:646 step:6K smpl:717K ep:16K epch:3.28 loss:0.051 grdn:0.241 lr:9.8e-05 updt_s:2.363 data_s:1.145 smp/s:36 mem_gb:52.79
|
| 45 |
+
Training: 10%|▉ | 5800/60000 [5:40:31<52:41:14, 3.50s/step]INFO 2026-07-17 21:29:10 ot_train.py:646 step:6K smpl:742K ep:17K epch:3.40 loss:0.052 grdn:0.251 lr:9.8e-05 updt_s:2.354 data_s:1.157 smp/s:36 mem_gb:52.71
|
| 46 |
+
Training: 10%|█ | 6000/60000 [5:52:12<52:32:43, 3.50s/step]INFO 2026-07-17 21:40:51 ot_train.py:646 step:6K smpl:768K ep:17K epch:3.52 loss:0.053 grdn:0.243 lr:9.8e-05 updt_s:2.357 data_s:1.148 smp/s:37 mem_gb:52.79
|
| 47 |
+
Training: 10%|█ | 6200/60000 [6:03:54<52:55:26, 3.54s/step]INFO 2026-07-17 21:52:33 ot_train.py:646 step:6K smpl:794K ep:18K epch:3.63 loss:0.050 grdn:0.230 lr:9.8e-05 updt_s:2.360 data_s:1.149 smp/s:36 mem_gb:52.79
|
| 48 |
+
Training: 11%|█ | 6400/60000 [6:15:37<52:08:13, 3.50s/step]INFO 2026-07-17 22:04:16 ot_train.py:646 step:6K smpl:819K ep:18K epch:3.75 loss:0.051 grdn:0.240 lr:9.8e-05 updt_s:2.364 data_s:1.151 smp/s:36 mem_gb:52.79
|
| 49 |
+
Training: 11%|█ | 6600/60000 [6:27:19<52:01:01, 3.51s/step]INFO 2026-07-17 22:15:58 ot_train.py:646 step:7K smpl:845K ep:19K epch:3.87 loss:0.052 grdn:0.237 lr:9.8e-05 updt_s:2.359 data_s:1.149 smp/s:36 mem_gb:52.79
|
| 50 |
+
Training: 11%|█▏ | 6800/60000 [6:39:02<53:07:11, 3.59s/step]INFO 2026-07-17 22:27:41 ot_train.py:646 step:7K smpl:870K ep:20K epch:3.99 loss:0.050 grdn:0.230 lr:9.7e-05 updt_s:2.352 data_s:1.157 smp/s:36 mem_gb:52.71
|
| 51 |
+
Training: 12%|█▏ | 7000/60000 [6:50:43<51:53:34, 3.52s/step]INFO 2026-07-17 22:39:22 ot_train.py:646 step:7K smpl:896K ep:20K epch:4.10 loss:0.047 grdn:0.215 lr:9.7e-05 updt_s:2.361 data_s:1.145 smp/s:37 mem_gb:52.79
|
| 52 |
+
Training: 12%|█▏ | 7200/60000 [7:02:26<51:08:34, 3.49s/step]INFO 2026-07-17 22:51:05 ot_train.py:646 step:7K smpl:922K ep:21K epch:4.22 loss:0.048 grdn:0.228 lr:9.7e-05 updt_s:2.363 data_s:1.148 smp/s:36 mem_gb:52.79
|
| 53 |
+
Training: 12%|█▏ | 7400/60000 [7:14:07<51:09:59, 3.50s/step]INFO 2026-07-17 23:02:46 ot_train.py:646 step:7K smpl:947K ep:21K epch:4.34 loss:0.049 grdn:0.226 lr:9.7e-05 updt_s:2.363 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 54 |
+
Training: 13%|█▎ | 7600/60000 [7:25:50<50:49:47, 3.49s/step]INFO 2026-07-17 23:14:29 ot_train.py:646 step:8K smpl:973K ep:22K epch:4.45 loss:0.048 grdn:0.216 lr:9.7e-05 updt_s:2.363 data_s:1.147 smp/s:36 mem_gb:52.79
|
| 55 |
+
Training: 13%|█▎ | 7800/60000 [7:37:32<50:54:18, 3.51s/step]INFO 2026-07-17 23:26:11 ot_train.py:646 step:8K smpl:998K ep:23K epch:4.57 loss:0.048 grdn:0.213 lr:9.6e-05 updt_s:2.367 data_s:1.145 smp/s:36 mem_gb:52.79
|
| 56 |
+
Training: 13%|█▎ | 8000/60000 [7:49:15<50:28:09, 3.49s/step]INFO 2026-07-17 23:37:54 ot_train.py:646 step:8K smpl:1M ep:23K epch:4.69 loss:0.048 grdn:0.217 lr:9.6e-05 updt_s:2.354 data_s:1.157 smp/s:36 mem_gb:52.71
|
| 57 |
+
Training: 14%|█▎ | 8200/60000 [8:01:00<50:11:36, 3.49s/step]INFO 2026-07-17 23:49:39 ot_train.py:646 step:8K smpl:1M ep:24K epch:4.81 loss:0.047 grdn:0.218 lr:9.6e-05 updt_s:2.367 data_s:1.155 smp/s:36 mem_gb:52.79
|
| 58 |
+
Training: 14%|█▍ | 8400/60000 [8:12:41<50:05:38, 3.49s/step]INFO 2026-07-18 00:01:20 ot_train.py:646 step:8K smpl:1M ep:24K epch:4.92 loss:0.046 grdn:0.211 lr:9.6e-05 updt_s:2.355 data_s:1.150 smp/s:37 mem_gb:52.79
|
| 59 |
+
Training: 14%|█▍ | 8600/60000 [8:24:23<49:56:32, 3.50s/step]INFO 2026-07-18 00:13:02 ot_train.py:646 step:9K smpl:1M ep:25K epch:5.04 loss:0.047 grdn:0.226 lr:9.6e-05 updt_s:2.361 data_s:1.148 smp/s:36 mem_gb:52.79
|
| 60 |
+
Training: 15%|█▍ | 8800/60000 [8:36:05<49:49:42, 3.50s/step]INFO 2026-07-18 00:24:44 ot_train.py:646 step:9K smpl:1M ep:25K epch:5.16 loss:0.047 grdn:0.218 lr:9.5e-05 updt_s:2.359 data_s:1.149 smp/s:36 mem_gb:52.79
|
| 61 |
+
Training: 15%|█▌ | 9000/60000 [8:47:48<50:33:30, 3.57s/step]INFO 2026-07-18 00:36:27 ot_train.py:646 step:9K smpl:1M ep:26K epch:5.28 loss:0.046 grdn:0.214 lr:9.5e-05 updt_s:2.362 data_s:1.149 smp/s:36 mem_gb:52.79
|
| 62 |
+
Training: 15%|█▌ | 9200/60000 [8:59:30<49:39:40, 3.52s/step]INFO 2026-07-18 00:48:09 ot_train.py:646 step:9K smpl:1M ep:27K epch:5.39 loss:0.045 grdn:0.204 lr:9.5e-05 updt_s:2.356 data_s:1.155 smp/s:36 mem_gb:52.71
|
| 63 |
+
Training: 16%|█▌ | 9400/60000 [9:11:13<48:54:27, 3.48s/step]INFO 2026-07-18 00:59:52 ot_train.py:646 step:9K smpl:1M ep:27K epch:5.51 loss:0.045 grdn:0.208 lr:9.5e-05 updt_s:2.367 data_s:1.146 smp/s:36 mem_gb:52.79
|
| 64 |
+
Training: 16%|█▌ | 9600/60000 [9:22:57<49:47:20, 3.56s/step]INFO 2026-07-18 01:11:36 ot_train.py:646 step:10K smpl:1M ep:28K epch:5.63 loss:0.045 grdn:0.202 lr:9.4e-05 updt_s:2.371 data_s:1.146 smp/s:36 mem_gb:52.79
|
| 65 |
+
Training: 16%|█▋ | 9800/60000 [9:34:40<49:08:31, 3.52s/step]INFO 2026-07-18 01:23:19 ot_train.py:646 step:10K smpl:1M ep:28K epch:5.74 loss:0.045 grdn:0.216 lr:9.4e-05 updt_s:2.368 data_s:1.146 smp/s:36 mem_gb:52.79
|
| 66 |
+
Training: 17%|█▋ | 10000/60000 [9:46:23<48:48:18, 3.51s/step]INFO 2026-07-18 01:35:02 ot_train.py:646 step:10K smpl:1M ep:29K epch:5.86 loss:0.044 grdn:0.203 lr:9.4e-05 updt_s:2.369 data_s:1.144 smp/s:36 mem_gb:52.79
|
| 67 |
+
Training: 17%|█▋ | 10200/60000 [9:58:06<48:43:20, 3.52s/step]INFO 2026-07-18 01:46:45 ot_train.py:646 step:10K smpl:1M ep:29K epch:5.98 loss:0.045 grdn:0.216 lr:9.4e-05 updt_s:2.360 data_s:1.153 smp/s:36 mem_gb:52.71
|
| 68 |
+
Training: 17%|█▋ | 10400/60000 [10:09:47<48:17:51, 3.51s/step]INFO 2026-07-18 01:58:26 ot_train.py:646 step:10K smpl:1M ep:30K epch:6.10 loss:0.043 grdn:0.208 lr:9.3e-05 updt_s:2.362 data_s:1.142 smp/s:37 mem_gb:52.79
|
| 69 |
+
Training: 18%|█▊ | 10600/60000 [10:21:27<48:35:25, 3.54s/step]INFO 2026-07-18 02:10:06 ot_train.py:646 step:11K smpl:1M ep:31K epch:6.21 loss:0.043 grdn:0.203 lr:9.3e-05 updt_s:2.357 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 70 |
+
Training: 18%|█▊ | 10800/60000 [10:33:07<47:45:59, 3.50s/step]INFO 2026-07-18 02:21:46 ot_train.py:646 step:11K smpl:1M ep:31K epch:6.33 loss:0.043 grdn:0.207 lr:9.3e-05 updt_s:2.358 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 71 |
+
Training: 18%|█▊ | 11000/60000 [10:44:49<47:25:01, 3.48s/step]INFO 2026-07-18 02:33:28 ot_train.py:646 step:11K smpl:1M ep:32K epch:6.45 loss:0.042 grdn:0.202 lr:9.3e-05 updt_s:2.361 data_s:1.146 smp/s:36 mem_gb:52.79
|
| 72 |
+
Training: 19%|█▊ | 11200/60000 [10:56:29<47:43:53, 3.52s/step]INFO 2026-07-18 02:45:08 ot_train.py:646 step:11K smpl:1M ep:32K epch:6.57 loss:0.042 grdn:0.204 lr:9.2e-05 updt_s:2.356 data_s:1.142 smp/s:37 mem_gb:52.79
|
| 73 |
+
Training: 19%|█▉ | 11400/60000 [11:08:08<47:23:30, 3.51s/step]INFO 2026-07-18 02:56:47 ot_train.py:646 step:11K smpl:1M ep:33K epch:6.68 loss:0.042 grdn:0.209 lr:9.2e-05 updt_s:2.347 data_s:1.146 smp/s:37 mem_gb:52.71
|
| 74 |
+
Training: 19%|█▉ | 11600/60000 [11:19:49<46:49:30, 3.48s/step]INFO 2026-07-18 03:08:28 ot_train.py:646 step:12K smpl:1M ep:34K epch:6.80 loss:0.041 grdn:0.211 lr:9.2e-05 updt_s:2.358 data_s:1.148 smp/s:37 mem_gb:52.79
|
| 75 |
+
Training: 20%|█▉ | 11800/60000 [11:31:30<47:05:20, 3.52s/step]INFO 2026-07-18 03:20:09 ot_train.py:646 step:12K smpl:2M ep:34K epch:6.92 loss:0.041 grdn:0.202 lr:9.2e-05 updt_s:2.358 data_s:1.145 smp/s:37 mem_gb:52.79
|
| 76 |
+
Training: 20%|██ | 12000/60000 [11:43:11<46:22:08, 3.48s/step]INFO 2026-07-18 03:31:50 ot_train.py:646 step:12K smpl:2M ep:35K epch:7.03 loss:0.041 grdn:0.201 lr:9.1e-05 updt_s:2.361 data_s:1.142 smp/s:37 mem_gb:52.79
|
| 77 |
+
Training: 20%|██ | 12200/60000 [11:54:55<46:48:17, 3.53s/step]INFO 2026-07-18 03:43:34 ot_train.py:646 step:12K smpl:2M ep:35K epch:7.15 loss:0.041 grdn:0.209 lr:9.1e-05 updt_s:2.367 data_s:1.148 smp/s:36 mem_gb:52.79
|
| 78 |
+
Training: 21%|██ | 12400/60000 [12:06:35<46:16:20, 3.50s/step]INFO 2026-07-18 03:55:14 ot_train.py:646 step:12K smpl:2M ep:36K epch:7.27 loss:0.042 grdn:0.213 lr:9.1e-05 updt_s:2.355 data_s:1.145 smp/s:37 mem_gb:52.79
|
| 79 |
+
Training: 21%|██ | 12600/60000 [12:18:17<45:44:14, 3.47s/step]INFO 2026-07-18 04:06:56 ot_train.py:646 step:13K smpl:2M ep:36K epch:7.39 loss:0.041 grdn:0.207 lr:9.0e-05 updt_s:2.355 data_s:1.151 smp/s:37 mem_gb:52.71
|
| 80 |
+
Training: 21%|██▏ | 12800/60000 [12:29:59<45:54:30, 3.50s/step]INFO 2026-07-18 04:18:38 ot_train.py:646 step:13K smpl:2M ep:37K epch:7.50 loss:0.040 grdn:0.202 lr:9.0e-05 updt_s:2.366 data_s:1.144 smp/s:36 mem_gb:52.79
|
| 81 |
+
Training: 22%|██▏ | 13000/60000 [12:41:41<45:48:34, 3.51s/step]INFO 2026-07-18 04:30:20 ot_train.py:646 step:13K smpl:2M ep:38K epch:7.62 loss:0.039 grdn:0.203 lr:9.0e-05 updt_s:2.362 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 82 |
+
Training: 22%|██▏ | 13200/60000 [12:53:22<45:25:39, 3.49s/step]INFO 2026-07-18 04:42:01 ot_train.py:646 step:13K smpl:2M ep:38K epch:7.74 loss:0.039 grdn:0.200 lr:8.9e-05 updt_s:2.359 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 83 |
+
Training: 22%|██▏ | 13400/60000 [13:05:04<45:06:39, 3.48s/step]INFO 2026-07-18 04:53:42 ot_train.py:646 step:13K smpl:2M ep:39K epch:7.85 loss:0.040 grdn:0.198 lr:8.9e-05 updt_s:2.361 data_s:1.147 smp/s:36 mem_gb:52.79
|
| 84 |
+
Training: 23%|██▎ | 13600/60000 [13:16:45<45:18:36, 3.52s/step]INFO 2026-07-18 05:05:24 ot_train.py:646 step:14K smpl:2M ep:39K epch:7.97 loss:0.039 grdn:0.208 lr:8.9e-05 updt_s:2.353 data_s:1.153 smp/s:37 mem_gb:52.71
|
| 85 |
+
Training: 23%|██▎ | 13800/60000 [13:28:27<44:39:19, 3.48s/step]INFO 2026-07-18 05:17:06 ot_train.py:646 step:14K smpl:2M ep:40K epch:8.09 loss:0.039 grdn:0.197 lr:8.8e-05 updt_s:2.367 data_s:1.144 smp/s:36 mem_gb:52.79
|
| 86 |
+
Training: 23%|██▎ | 14000/60000 [13:40:09<44:26:54, 3.48s/step]INFO 2026-07-18 05:28:48 ot_train.py:646 step:14K smpl:2M ep:40K epch:8.21 loss:0.038 grdn:0.198 lr:8.8e-05 updt_s:2.365 data_s:1.144 smp/s:36 mem_gb:52.79
|
| 87 |
+
Training: 24%|██▎ | 14200/60000 [13:51:51<44:12:47, 3.48s/step]INFO 2026-07-18 05:40:30 ot_train.py:646 step:14K smpl:2M ep:41K epch:8.32 loss:0.039 grdn:0.198 lr:8.8e-05 updt_s:2.361 data_s:1.147 smp/s:36 mem_gb:52.79
|
| 88 |
+
Training: 24%|██▍ | 14400/60000 [14:03:34<44:22:55, 3.50s/step]INFO 2026-07-18 05:52:13 ot_train.py:646 step:14K smpl:2M ep:42K epch:8.44 loss:0.039 grdn:0.205 lr:8.7e-05 updt_s:2.364 data_s:1.145 smp/s:36 mem_gb:52.79
|
| 89 |
+
Training: 24%|██▍ | 14600/60000 [14:15:15<44:40:07, 3.54s/step]INFO 2026-07-18 06:03:54 ot_train.py:646 step:15K smpl:2M ep:42K epch:8.56 loss:0.040 grdn:0.204 lr:8.7e-05 updt_s:2.358 data_s:1.148 smp/s:37 mem_gb:52.79
|
| 90 |
+
Training: 25%|██▍ | 14800/60000 [14:26:55<43:33:40, 3.47s/step]INFO 2026-07-18 06:15:34 ot_train.py:646 step:15K smpl:2M ep:43K epch:8.68 loss:0.038 grdn:0.200 lr:8.7e-05 updt_s:2.345 data_s:1.150 smp/s:37 mem_gb:52.71
|
| 91 |
+
Training: 25%|██▌ | 15000/60000 [14:38:34<43:28:06, 3.48s/step]INFO 2026-07-18 06:27:13 ot_train.py:646 step:15K smpl:2M ep:43K epch:8.79 loss:0.038 grdn:0.207 lr:8.6e-05 updt_s:2.356 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 92 |
+
Training: 25%|██▌ | 15200/60000 [14:50:14<43:34:33, 3.50s/step]INFO 2026-07-18 06:38:53 ot_train.py:646 step:15K smpl:2M ep:44K epch:8.91 loss:0.036 grdn:0.192 lr:8.6e-05 updt_s:2.357 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 93 |
+
Training: 26%|██▌ | 15400/60000 [15:01:56<43:33:11, 3.52s/step]INFO 2026-07-18 06:50:35 ot_train.py:646 step:15K smpl:2M ep:45K epch:9.03 loss:0.037 grdn:0.195 lr:8.5e-05 updt_s:2.365 data_s:1.142 smp/s:36 mem_gb:52.79
|
| 94 |
+
Training: 26%|██▌ | 15600/60000 [15:13:36<43:15:44, 3.51s/step]INFO 2026-07-18 07:02:15 ot_train.py:646 step:16K smpl:2M ep:45K epch:9.14 loss:0.038 grdn:0.202 lr:8.5e-05 updt_s:2.357 data_s:1.143 smp/s:37 mem_gb:52.79
|
| 95 |
+
Training: 26%|██▋ | 15800/60000 [15:25:16<42:29:26, 3.46s/step]INFO 2026-07-18 07:13:55 ot_train.py:646 step:16K smpl:2M ep:46K epch:9.26 loss:0.037 grdn:0.200 lr:8.5e-05 updt_s:2.354 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 96 |
+
Training: 27%|██▋ | 16000/60000 [15:36:57<42:37:05, 3.49s/step]INFO 2026-07-18 07:25:36 ot_train.py:646 step:16K smpl:2M ep:46K epch:9.38 loss:0.036 grdn:0.190 lr:8.4e-05 updt_s:2.355 data_s:1.151 smp/s:37 mem_gb:52.71
|
| 97 |
+
Training: 27%|██▋ | 16200/60000 [15:48:38<42:46:14, 3.52s/step]INFO 2026-07-18 07:37:16 ot_train.py:646 step:16K smpl:2M ep:47K epch:9.50 loss:0.036 grdn:0.194 lr:8.4e-05 updt_s:2.358 data_s:1.143 smp/s:37 mem_gb:52.79
|
| 98 |
+
Training: 27%|██▋ | 16400/60000 [16:00:19<42:08:11, 3.48s/step]INFO 2026-07-18 07:48:58 ot_train.py:646 step:16K smpl:2M ep:47K epch:9.61 loss:0.037 grdn:0.198 lr:8.4e-05 updt_s:2.360 data_s:1.145 smp/s:37 mem_gb:52.79
|
| 99 |
+
Training: 28%|██▊ | 16600/60000 [16:12:00<42:03:53, 3.49s/step]INFO 2026-07-18 08:00:39 ot_train.py:646 step:17K smpl:2M ep:48K epch:9.73 loss:0.036 grdn:0.193 lr:8.3e-05 updt_s:2.361 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 100 |
+
Training: 28%|██▊ | 16800/60000 [16:23:42<42:03:45, 3.51s/step]INFO 2026-07-18 08:12:21 ot_train.py:646 step:17K smpl:2M ep:49K epch:9.85 loss:0.036 grdn:0.198 lr:8.3e-05 updt_s:2.363 data_s:1.143 smp/s:37 mem_gb:52.79
|
| 101 |
+
Training: 28%|██▊ | 17000/60000 [16:35:23<41:40:02, 3.49s/step]INFO 2026-07-18 08:24:02 ot_train.py:646 step:17K smpl:2M ep:49K epch:9.96 loss:0.036 grdn:0.210 lr:8.2e-05 updt_s:2.351 data_s:1.152 smp/s:37 mem_gb:52.71
|
| 102 |
+
Training: 29%|██▊ | 17200/60000 [16:47:05<41:25:32, 3.48s/step]INFO 2026-07-18 08:35:44 ot_train.py:646 step:17K smpl:2M ep:50K epch:10.08 loss:0.036 grdn:0.200 lr:8.2e-05 updt_s:2.367 data_s:1.142 smp/s:36 mem_gb:52.79
|
| 103 |
+
Training: 29%|██▉ | 17400/60000 [16:58:46<41:17:48, 3.49s/step]INFO 2026-07-18 08:47:25 ot_train.py:646 step:17K smpl:2M ep:50K epch:10.20 loss:0.035 grdn:0.210 lr:8.2e-05 updt_s:2.364 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 104 |
+
Training: 29%|██▉ | 17600/60000 [17:10:28<41:12:14, 3.50s/step]INFO 2026-07-18 08:59:07 ot_train.py:646 step:18K smpl:2M ep:51K epch:10.32 loss:0.034 grdn:0.192 lr:8.1e-05 updt_s:2.368 data_s:1.143 smp/s:36 mem_gb:52.79
|
| 105 |
+
Training: 30%|██▉ | 17800/60000 [17:22:09<40:44:31, 3.48s/step]INFO 2026-07-18 09:10:47 ot_train.py:646 step:18K smpl:2M ep:51K epch:10.43 loss:0.034 grdn:0.189 lr:8.1e-05 updt_s:2.358 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 106 |
+
Training: 30%|███ | 18000/60000 [17:33:49<40:46:18, 3.49s/step]INFO 2026-07-18 09:22:28 ot_train.py:646 step:18K smpl:2M ep:52K epch:10.55 loss:0.034 grdn:0.189 lr:8.0e-05 updt_s:2.361 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 107 |
+
Training: 30%|███ | 18200/60000 [17:45:30<40:42:39, 3.51s/step]INFO 2026-07-18 09:34:09 ot_train.py:646 step:18K smpl:2M ep:53K epch:10.67 loss:0.034 grdn:0.204 lr:8.0e-05 updt_s:2.357 data_s:1.146 smp/s:37 mem_gb:52.71
|
| 108 |
+
Training: 31%|███ | 18400/60000 [17:57:11<40:20:32, 3.49s/step]INFO 2026-07-18 09:45:50 ot_train.py:646 step:18K smpl:2M ep:53K epch:10.79 loss:0.033 grdn:0.197 lr:7.9e-05 updt_s:2.367 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 109 |
+
Training: 31%|███ | 18600/60000 [18:08:52<40:07:55, 3.49s/step]INFO 2026-07-18 09:57:31 ot_train.py:646 step:19K smpl:2M ep:54K epch:10.90 loss:0.033 grdn:0.205 lr:7.9e-05 updt_s:2.365 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 110 |
+
Training: 31%|███▏ | 18800/60000 [18:20:32<40:09:57, 3.51s/step]INFO 2026-07-18 10:09:11 ot_train.py:646 step:19K smpl:2M ep:54K epch:11.02 loss:0.034 grdn:0.194 lr:7.9e-05 updt_s:2.359 data_s:1.142 smp/s:37 mem_gb:52.79
|
| 111 |
+
Training: 32%|███▏ | 19000/60000 [18:32:12<39:46:27, 3.49s/step]INFO 2026-07-18 10:20:51 ot_train.py:646 step:19K smpl:2M ep:55K epch:11.14 loss:0.033 grdn:0.198 lr:7.8e-05 updt_s:2.356 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 112 |
+
Training: 32%|███▏ | 19200/60000 [18:43:53<41:38:00, 3.67s/step]INFO 2026-07-18 10:32:32 ot_train.py:646 step:19K smpl:2M ep:55K epch:11.25 loss:0.033 grdn:0.195 lr:7.8e-05 updt_s:2.353 data_s:1.149 smp/s:37 mem_gb:52.71
|
| 113 |
+
Training: 32%|███▏ | 19400/60000 [18:55:33<39:28:50, 3.50s/step]INFO 2026-07-18 10:44:12 ot_train.py:646 step:19K smpl:2M ep:56K epch:11.37 loss:0.033 grdn:0.204 lr:7.7e-05 updt_s:2.363 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 114 |
+
Training: 33%|███▎ | 19600/60000 [19:07:12<38:58:54, 3.47s/step]INFO 2026-07-18 10:55:51 ot_train.py:646 step:20K smpl:3M ep:57K epch:11.49 loss:0.032 grdn:0.200 lr:7.7e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 115 |
+
Training: 33%|███▎ | 19800/60000 [19:18:53<39:04:03, 3.50s/step]INFO 2026-07-18 11:07:32 ot_train.py:646 step:20K smpl:3M ep:57K epch:11.61 loss:0.033 grdn:0.203 lr:7.6e-05 updt_s:2.362 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 116 |
+
Training: 33%|███▎ | 20000/60000 [19:30:35<38:37:54, 3.48s/step]INFO 2026-07-18 11:19:14 ot_train.py:646 step:20K smpl:3M ep:58K epch:11.72 loss:0.033 grdn:0.201 lr:7.6e-05 updt_s:2.368 data_s:1.140 smp/s:36 mem_gb:52.79
|
| 117 |
+
Training: 34%|███▎ | 20200/60000 [19:42:18<38:39:11, 3.50s/step]INFO 2026-07-18 11:30:57 ot_train.py:646 step:20K smpl:3M ep:58K epch:11.84 loss:0.033 grdn:0.197 lr:7.6e-05 updt_s:2.365 data_s:1.151 smp/s:36 mem_gb:52.79
|
| 118 |
+
Training: 34%|███▍ | 20400/60000 [19:53:57<38:28:49, 3.50s/step]INFO 2026-07-18 11:42:36 ot_train.py:646 step:20K smpl:3M ep:59K epch:11.96 loss:0.032 grdn:0.198 lr:7.5e-05 updt_s:2.351 data_s:1.141 smp/s:37 mem_gb:52.71
|
| 119 |
+
Training: 34%|███▍ | 20600/60000 [20:05:36<38:15:38, 3.50s/step]INFO 2026-07-18 11:54:14 ot_train.py:646 step:21K smpl:3M ep:60K epch:12.08 loss:0.031 grdn:0.198 lr:7.5e-05 updt_s:2.355 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 120 |
+
Training: 35%|███▍ | 20800/60000 [20:17:16<38:11:32, 3.51s/step]INFO 2026-07-18 12:05:55 ot_train.py:646 step:21K smpl:3M ep:60K epch:12.19 loss:0.031 grdn:0.194 lr:7.4e-05 updt_s:2.361 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 121 |
+
Training: 35%|███▌ | 21000/60000 [20:28:54<37:49:19, 3.49s/step]INFO 2026-07-18 12:17:33 ot_train.py:646 step:21K smpl:3M ep:61K epch:12.31 loss:0.031 grdn:0.195 lr:7.4e-05 updt_s:2.353 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 122 |
+
Training: 35%|███▌ | 21200/60000 [20:40:34<37:42:59, 3.50s/step]INFO 2026-07-18 12:29:13 ot_train.py:646 step:21K smpl:3M ep:61K epch:12.43 loss:0.032 grdn:0.196 lr:7.3e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 123 |
+
Training: 36%|███▌ | 21400/60000 [20:52:12<37:26:57, 3.49s/step]INFO 2026-07-18 12:40:51 ot_train.py:646 step:21K smpl:3M ep:62K epch:12.54 loss:0.031 grdn:0.195 lr:7.3e-05 updt_s:2.354 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 124 |
+
Training: 36%|███▌ | 21600/60000 [21:03:52<36:56:06, 3.46s/step]INFO 2026-07-18 12:52:31 ot_train.py:646 step:22K smpl:3M ep:62K epch:12.66 loss:0.030 grdn:0.196 lr:7.2e-05 updt_s:2.346 data_s:1.149 smp/s:37 mem_gb:52.71
|
| 125 |
+
Training: 36%|███▋ | 21800/60000 [21:15:30<36:55:06, 3.48s/step]INFO 2026-07-18 13:04:09 ot_train.py:646 step:22K smpl:3M ep:63K epch:12.78 loss:0.030 grdn:0.204 lr:7.2e-05 updt_s:2.355 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 126 |
+
Training: 37%|███▋ | 22000/60000 [21:27:09<36:42:26, 3.48s/step]INFO 2026-07-18 13:15:48 ot_train.py:646 step:22K smpl:3M ep:64K epch:12.90 loss:0.031 grdn:0.205 lr:7.1e-05 updt_s:2.356 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 127 |
+
Training: 37%|███▋ | 22200/60000 [21:38:49<36:44:20, 3.50s/step]INFO 2026-07-18 13:27:28 ot_train.py:646 step:22K smpl:3M ep:64K epch:13.01 loss:0.030 grdn:0.197 lr:7.1e-05 updt_s:2.359 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 128 |
+
Training: 37%|███▋ | 22400/60000 [21:50:29<36:52:05, 3.53s/step]INFO 2026-07-18 13:39:08 ot_train.py:646 step:22K smpl:3M ep:65K epch:13.13 loss:0.030 grdn:0.197 lr:7.0e-05 updt_s:2.362 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 129 |
+
Training: 38%|███▊ | 22600/60000 [22:02:09<36:30:25, 3.51s/step]INFO 2026-07-18 13:50:48 ot_train.py:646 step:23K smpl:3M ep:65K epch:13.25 loss:0.029 grdn:0.196 lr:7.0e-05 updt_s:2.351 data_s:1.148 smp/s:37 mem_gb:52.71
|
| 130 |
+
Training: 38%|███▊ | 22800/60000 [22:13:47<36:10:33, 3.50s/step]INFO 2026-07-18 14:02:26 ot_train.py:646 step:23K smpl:3M ep:66K epch:13.36 loss:0.029 grdn:0.198 lr:6.9e-05 updt_s:2.354 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 131 |
+
Training: 38%|███▊ | 23000/60000 [22:25:27<35:47:07, 3.48s/step]INFO 2026-07-18 14:14:06 ot_train.py:646 step:23K smpl:3M ep:66K epch:13.48 loss:0.028 grdn:0.191 lr:6.9e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 132 |
+
Training: 39%|███▊ | 23200/60000 [22:37:06<35:41:50, 3.49s/step]INFO 2026-07-18 14:25:45 ot_train.py:646 step:23K smpl:3M ep:67K epch:13.60 loss:0.028 grdn:0.198 lr:6.8e-05 updt_s:2.359 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 133 |
+
Training: 39%|███▉ | 23400/60000 [22:48:45<35:32:33, 3.50s/step]INFO 2026-07-18 14:37:24 ot_train.py:646 step:23K smpl:3M ep:68K epch:13.72 loss:0.028 grdn:0.192 lr:6.8e-05 updt_s:2.357 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 134 |
+
Training: 39%|███▉ | 23600/60000 [23:00:24<35:12:50, 3.48s/step]INFO 2026-07-18 14:49:03 ot_train.py:646 step:24K smpl:3M ep:68K epch:13.83 loss:0.029 grdn:0.192 lr:6.7e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 135 |
+
Training: 40%|███▉ | 23800/60000 [23:12:04<35:03:19, 3.49s/step]INFO 2026-07-18 15:00:43 ot_train.py:646 step:24K smpl:3M ep:69K epch:13.95 loss:0.029 grdn:0.201 lr:6.7e-05 updt_s:2.349 data_s:1.146 smp/s:37 mem_gb:52.71
|
| 136 |
+
Training: 40%|████ | 24000/60000 [23:23:42<34:55:14, 3.49s/step]INFO 2026-07-18 15:12:21 ot_train.py:646 step:24K smpl:3M ep:69K epch:14.07 loss:0.027 grdn:0.193 lr:6.6e-05 updt_s:2.355 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 137 |
+
Training: 40%|████ | 24200/60000 [23:35:23<34:44:20, 3.49s/step]INFO 2026-07-18 15:24:02 ot_train.py:646 step:24K smpl:3M ep:70K epch:14.19 loss:0.028 grdn:0.200 lr:6.6e-05 updt_s:2.363 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 138 |
+
Training: 41%|████ | 24400/60000 [23:47:03<34:17:35, 3.47s/step]INFO 2026-07-18 15:35:42 ot_train.py:646 step:24K smpl:3M ep:71K epch:14.30 loss:0.028 grdn:0.195 lr:6.5e-05 updt_s:2.364 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 139 |
+
Training: 41%|████ | 24600/60000 [23:58:42<34:18:20, 3.49s/step]INFO 2026-07-18 15:47:21 ot_train.py:646 step:25K smpl:3M ep:71K epch:14.42 loss:0.028 grdn:0.200 lr:6.5e-05 updt_s:2.360 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 140 |
+
Training: 41%|████▏ | 24800/60000 [24:10:22<34:55:21, 3.57s/step]INFO 2026-07-18 15:59:01 ot_train.py:646 step:25K smpl:3M ep:72K epch:14.54 loss:0.028 grdn:0.201 lr:6.4e-05 updt_s:2.361 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 141 |
+
Training: 42%|████▏ | 25000/60000 [24:22:01<33:50:33, 3.48s/step]INFO 2026-07-18 16:10:40 ot_train.py:646 step:25K smpl:3M ep:72K epch:14.65 loss:0.027 grdn:0.192 lr:6.4e-05 updt_s:2.348 data_s:1.148 smp/s:37 mem_gb:52.71
|
| 142 |
+
Training: 42%|████▏ | 25200/60000 [24:33:41<33:36:15, 3.48s/step]INFO 2026-07-18 16:22:20 ot_train.py:646 step:25K smpl:3M ep:73K epch:14.77 loss:0.026 grdn:0.187 lr:6.3e-05 updt_s:2.357 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 143 |
+
Training: 42%|████▏ | 25400/60000 [24:45:21<33:22:31, 3.47s/step]INFO 2026-07-18 16:34:00 ot_train.py:646 step:25K smpl:3M ep:73K epch:14.89 loss:0.027 grdn:0.199 lr:6.3e-05 updt_s:2.359 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 144 |
+
Training: 43%|████▎ | 25600/60000 [24:57:00<33:22:01, 3.49s/step]INFO 2026-07-18 16:45:39 ot_train.py:646 step:26K smpl:3M ep:74K epch:15.01 loss:0.027 grdn:0.193 lr:6.2e-05 updt_s:2.356 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 145 |
+
Training: 43%|████▎ | 25800/60000 [25:08:39<33:14:07, 3.50s/step]INFO 2026-07-18 16:57:18 ot_train.py:646 step:26K smpl:3M ep:75K epch:15.12 loss:0.026 grdn:0.189 lr:6.2e-05 updt_s:2.356 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 146 |
+
Training: 43%|████▎ | 26000/60000 [25:20:20<33:19:34, 3.53s/step]INFO 2026-07-18 17:08:59 ot_train.py:646 step:26K smpl:3M ep:75K epch:15.24 loss:0.026 grdn:0.192 lr:6.1e-05 updt_s:2.358 data_s:1.147 smp/s:37 mem_gb:52.71
|
| 147 |
+
Training: 44%|████▎ | 26200/60000 [25:32:01<33:02:24, 3.52s/step]INFO 2026-07-18 17:20:40 ot_train.py:646 step:26K smpl:3M ep:76K epch:15.36 loss:0.025 grdn:0.193 lr:6.1e-05 updt_s:2.360 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 148 |
+
Training: 44%|████▍ | 26400/60000 [25:43:40<32:34:24, 3.49s/step]INFO 2026-07-18 17:32:19 ot_train.py:646 step:26K smpl:3M ep:76K epch:15.47 loss:0.026 grdn:0.196 lr:6.0e-05 updt_s:2.356 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 149 |
+
Training: 44%|████▍ | 26600/60000 [25:55:19<32:28:31, 3.50s/step]INFO 2026-07-18 17:43:58 ot_train.py:646 step:27K smpl:3M ep:77K epch:15.59 loss:0.025 grdn:0.191 lr:6.0e-05 updt_s:2.355 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 150 |
+
Training: 45%|████▍ | 26800/60000 [26:06:59<32:08:02, 3.48s/step]INFO 2026-07-18 17:55:38 ot_train.py:646 step:27K smpl:3M ep:77K epch:15.71 loss:0.025 grdn:0.184 lr:5.9e-05 updt_s:2.360 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 151 |
+
Training: 45%|████▌ | 27000/60000 [26:18:40<31:57:35, 3.49s/step]INFO 2026-07-18 18:07:19 ot_train.py:646 step:27K smpl:3M ep:78K epch:15.83 loss:0.026 grdn:0.187 lr:5.9e-05 updt_s:2.364 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 152 |
+
Training: 45%|████▌ | 27200/60000 [26:30:20<31:43:29, 3.48s/step]INFO 2026-07-18 18:18:59 ot_train.py:646 step:27K smpl:3M ep:79K epch:15.94 loss:0.025 grdn:0.190 lr:5.8e-05 updt_s:2.356 data_s:1.144 smp/s:37 mem_gb:52.71
|
| 153 |
+
Training: 46%|████▌ | 27400/60000 [26:42:00<31:27:02, 3.47s/step]INFO 2026-07-18 18:30:38 ot_train.py:646 step:27K smpl:4M ep:79K epch:16.06 loss:0.024 grdn:0.189 lr:5.8e-05 updt_s:2.358 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 154 |
+
Training: 46%|████▌ | 27600/60000 [26:53:41<31:22:41, 3.49s/step]INFO 2026-07-18 18:42:19 ot_train.py:646 step:28K smpl:4M ep:80K epch:16.18 loss:0.025 grdn:0.186 lr:5.7e-05 updt_s:2.363 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 155 |
+
Training: 46%|████▋ | 27800/60000 [27:05:20<31:08:37, 3.48s/step]INFO 2026-07-18 18:53:59 ot_train.py:646 step:28K smpl:4M ep:80K epch:16.30 loss:0.024 grdn:0.187 lr:5.7e-05 updt_s:2.356 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 156 |
+
Training: 47%|████▋ | 28000/60000 [27:16:59<31:07:55, 3.50s/step]INFO 2026-07-18 19:05:38 ot_train.py:646 step:28K smpl:4M ep:81K epch:16.41 loss:0.024 grdn:0.193 lr:5.6e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 157 |
+
Training: 47%|████▋ | 28200/60000 [27:28:41<30:41:38, 3.47s/step]INFO 2026-07-18 19:17:20 ot_train.py:646 step:28K smpl:4M ep:81K epch:16.53 loss:0.024 grdn:0.193 lr:5.6e-05 updt_s:2.366 data_s:1.142 smp/s:36 mem_gb:52.79
|
| 158 |
+
Training: 47%|████▋ | 28400/60000 [27:40:21<30:36:21, 3.49s/step]INFO 2026-07-18 19:29:00 ot_train.py:646 step:28K smpl:4M ep:82K epch:16.65 loss:0.024 grdn:0.191 lr:5.5e-05 updt_s:2.350 data_s:1.149 smp/s:37 mem_gb:52.71
|
| 159 |
+
Training: 48%|████▊ | 28600/60000 [27:52:01<30:19:17, 3.48s/step]INFO 2026-07-18 19:40:40 ot_train.py:646 step:29K smpl:4M ep:83K epch:16.76 loss:0.023 grdn:0.182 lr:5.5e-05 updt_s:2.357 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 160 |
+
Training: 48%|████▊ | 28800/60000 [28:03:41<30:21:29, 3.50s/step]INFO 2026-07-18 19:52:20 ot_train.py:646 step:29K smpl:4M ep:83K epch:16.88 loss:0.023 grdn:0.185 lr:5.4e-05 updt_s:2.354 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 161 |
+
Training: 48%|████▊ | 29000/60000 [28:15:21<29:56:06, 3.48s/step]INFO 2026-07-18 20:04:00 ot_train.py:646 step:29K smpl:4M ep:84K epch:17.00 loss:0.023 grdn:0.178 lr:5.4e-05 updt_s:2.359 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 162 |
+
Training: 49%|████▊ | 29200/60000 [28:27:02<29:51:44, 3.49s/step]INFO 2026-07-18 20:15:41 ot_train.py:646 step:29K smpl:4M ep:84K epch:17.12 loss:0.023 grdn:0.184 lr:5.3e-05 updt_s:2.366 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 163 |
+
Training: 49%|████▉ | 29400/60000 [28:38:41<29:39:14, 3.49s/step]INFO 2026-07-18 20:27:20 ot_train.py:646 step:29K smpl:4M ep:85K epch:17.23 loss:0.023 grdn:0.188 lr:5.3e-05 updt_s:2.347 data_s:1.146 smp/s:37 mem_gb:52.71
|
| 164 |
+
Training: 49%|████▉ | 29600/60000 [28:50:20<29:39:11, 3.51s/step]INFO 2026-07-18 20:38:59 ot_train.py:646 step:30K smpl:4M ep:86K epch:17.35 loss:0.022 grdn:0.187 lr:5.2e-05 updt_s:2.356 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 165 |
+
Training: 50%|████▉ | 29800/60000 [29:01:59<29:22:18, 3.50s/step]INFO 2026-07-18 20:50:38 ot_train.py:646 step:30K smpl:4M ep:86K epch:17.47 loss:0.022 grdn:0.184 lr:5.1e-05 updt_s:2.355 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 166 |
+
Training: 50%|█████ | 30000/60000 [29:13:39<28:57:38, 3.48s/step]INFO 2026-07-18 21:02:18 ot_train.py:646 step:30K smpl:4M ep:87K epch:17.59 loss:0.022 grdn:0.188 lr:5.1e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 167 |
+
Training: 50%|█████ | 30200/60000 [29:25:18<28:59:41, 3.50s/step]INFO 2026-07-18 21:13:57 ot_train.py:646 step:30K smpl:4M ep:87K epch:17.70 loss:0.022 grdn:0.181 lr:5.0e-05 updt_s:2.356 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 168 |
+
Training: 51%|█████ | 30400/60000 [29:36:58<28:58:43, 3.52s/step]INFO 2026-07-18 21:25:37 ot_train.py:646 step:30K smpl:4M ep:88K epch:17.82 loss:0.022 grdn:0.191 lr:5.0e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 169 |
+
Training: 51%|█████ | 30600/60000 [29:48:36<28:34:13, 3.50s/step]INFO 2026-07-18 21:37:15 ot_train.py:646 step:31K smpl:4M ep:88K epch:17.94 loss:0.021 grdn:0.179 lr:4.9e-05 updt_s:2.350 data_s:1.142 smp/s:37 mem_gb:52.71
|
| 170 |
+
Training: 51%|█████▏ | 30800/60000 [30:00:15<28:14:43, 3.48s/step]INFO 2026-07-18 21:48:54 ot_train.py:646 step:31K smpl:4M ep:89K epch:18.05 loss:0.021 grdn:0.186 lr:4.9e-05 updt_s:2.355 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 171 |
+
Training: 52%|█████▏ | 31000/60000 [30:11:55<28:07:42, 3.49s/step]INFO 2026-07-18 22:00:34 ot_train.py:646 step:31K smpl:4M ep:90K epch:18.17 loss:0.021 grdn:0.186 lr:4.8e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 172 |
+
Training: 52%|█████▏ | 31200/60000 [30:23:35<27:45:41, 3.47s/step]INFO 2026-07-18 22:12:14 ot_train.py:646 step:31K smpl:4M ep:90K epch:18.29 loss:0.021 grdn:0.184 lr:4.8e-05 updt_s:2.361 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 173 |
+
Training: 52%|█████▏ | 31400/60000 [30:35:15<27:36:15, 3.47s/step]INFO 2026-07-18 22:23:54 ot_train.py:646 step:31K smpl:4M ep:91K epch:18.41 loss:0.021 grdn:0.171 lr:4.7e-05 updt_s:2.355 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 174 |
+
Training: 53%|█████▎ | 31600/60000 [30:46:54<27:21:09, 3.47s/step]INFO 2026-07-18 22:35:33 ot_train.py:646 step:32K smpl:4M ep:91K epch:18.52 loss:0.021 grdn:0.178 lr:4.7e-05 updt_s:2.357 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 175 |
+
Training: 53%|█████▎ | 31800/60000 [30:58:34<28:04:45, 3.58s/step]INFO 2026-07-18 22:47:13 ot_train.py:646 step:32K smpl:4M ep:92K epch:18.64 loss:0.020 grdn:0.178 lr:4.6e-05 updt_s:2.350 data_s:1.145 smp/s:37 mem_gb:52.71
|
| 176 |
+
Training: 53%|█████▎ | 32000/60000 [31:10:13<26:58:01, 3.47s/step]INFO 2026-07-18 22:58:52 ot_train.py:646 step:32K smpl:4M ep:92K epch:18.76 loss:0.020 grdn:0.179 lr:4.6e-05 updt_s:2.358 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 177 |
+
Training: 54%|█████▎ | 32200/60000 [31:21:52<26:49:36, 3.47s/step]INFO 2026-07-18 23:10:31 ot_train.py:646 step:32K smpl:4M ep:93K epch:18.87 loss:0.020 grdn:0.174 lr:4.5e-05 updt_s:2.358 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 178 |
+
Training: 54%|█████▍ | 32400/60000 [31:33:30<27:00:58, 3.52s/step]INFO 2026-07-18 23:22:08 ot_train.py:646 step:32K smpl:4M ep:94K epch:18.99 loss:0.020 grdn:0.179 lr:4.5e-05 updt_s:2.352 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 179 |
+
Training: 54%|█████▍ | 32600/60000 [31:45:07<26:24:06, 3.47s/step]INFO 2026-07-18 23:33:46 ot_train.py:646 step:33K smpl:4M ep:94K epch:19.11 loss:0.020 grdn:0.175 lr:4.4e-05 updt_s:2.353 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 180 |
+
Training: 55%|█████▍ | 32800/60000 [31:56:46<26:16:12, 3.48s/step]INFO 2026-07-18 23:45:24 ot_train.py:646 step:33K smpl:4M ep:95K epch:19.23 loss:0.019 grdn:0.165 lr:4.4e-05 updt_s:2.347 data_s:1.142 smp/s:37 mem_gb:52.71
|
| 181 |
+
Training: 55%|█████▌ | 33000/60000 [32:08:25<26:07:07, 3.48s/step]INFO 2026-07-18 23:57:04 ot_train.py:646 step:33K smpl:4M ep:95K epch:19.34 loss:0.019 grdn:0.170 lr:4.3e-05 updt_s:2.361 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 182 |
+
Training: 55%|█████▌ | 33200/60000 [32:20:04<25:50:57, 3.47s/step]INFO 2026-07-19 00:08:42 ot_train.py:646 step:33K smpl:4M ep:96K epch:19.46 loss:0.019 grdn:0.178 lr:4.3e-05 updt_s:2.359 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 183 |
+
Training: 56%|█████▌ | 33400/60000 [32:31:42<25:46:49, 3.49s/step]INFO 2026-07-19 00:20:21 ot_train.py:646 step:33K smpl:4M ep:97K epch:19.58 loss:0.019 grdn:0.166 lr:4.2e-05 updt_s:2.359 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 184 |
+
Training: 56%|█████▌ | 33600/60000 [32:43:21<25:34:28, 3.49s/step]INFO 2026-07-19 00:32:00 ot_train.py:646 step:34K smpl:4M ep:97K epch:19.70 loss:0.019 grdn:0.171 lr:4.1e-05 updt_s:2.356 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 185 |
+
Training: 56%|█████▋ | 33800/60000 [32:55:01<25:19:38, 3.48s/step]INFO 2026-07-19 00:43:40 ot_train.py:646 step:34K smpl:4M ep:98K epch:19.81 loss:0.019 grdn:0.177 lr:4.1e-05 updt_s:2.362 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 186 |
+
Training: 57%|█████▋ | 34000/60000 [33:06:40<25:51:34, 3.58s/step]INFO 2026-07-19 00:55:19 ot_train.py:646 step:34K smpl:4M ep:98K epch:19.93 loss:0.019 grdn:0.177 lr:4.0e-05 updt_s:2.354 data_s:1.140 smp/s:37 mem_gb:52.71
|
| 187 |
+
Training: 57%|█████▋ | 34200/60000 [33:18:20<25:04:37, 3.50s/step]INFO 2026-07-19 01:06:59 ot_train.py:646 step:34K smpl:4M ep:99K epch:20.05 loss:0.018 grdn:0.162 lr:4.0e-05 updt_s:2.365 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 188 |
+
Training: 57%|█████▋ | 34400/60000 [33:29:59<24:43:44, 3.48s/step]INFO 2026-07-19 01:18:37 ot_train.py:646 step:34K smpl:4M ep:99K epch:20.16 loss:0.018 grdn:0.176 lr:3.9e-05 updt_s:2.358 data_s:1.132 smp/s:37 mem_gb:52.79
|
| 189 |
+
Training: 58%|█████▊ | 34600/60000 [33:41:38<24:38:22, 3.49s/step]INFO 2026-07-19 01:30:17 ot_train.py:646 step:35K smpl:4M ep:100K epch:20.28 loss:0.018 grdn:0.176 lr:3.9e-05 updt_s:2.363 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 190 |
+
Training: 58%|█████▊ | 34800/60000 [33:53:18<24:47:44, 3.54s/step]INFO 2026-07-19 01:41:57 ot_train.py:646 step:35K smpl:4M ep:101K epch:20.40 loss:0.018 grdn:0.175 lr:3.8e-05 updt_s:2.361 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 191 |
+
Training: 58%|█████▊ | 35000/60000 [34:04:54<25:18:25, 3.64s/step]INFO 2026-07-19 01:53:33 ot_train.py:646 step:35K smpl:4M ep:101K epch:20.52 loss:0.018 grdn:0.173 lr:3.8e-05 updt_s:2.339 data_s:1.140 smp/s:37 mem_gb:52.71
|
| 192 |
+
Training: 59%|█████▊ | 35200/60000 [34:16:30<23:58:36, 3.48s/step]INFO 2026-07-19 02:05:09 ot_train.py:646 step:35K smpl:5M ep:102K epch:20.63 loss:0.017 grdn:0.169 lr:3.7e-05 updt_s:2.339 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 193 |
+
Training: 59%|█████▉ | 35400/60000 [34:28:07<23:45:31, 3.48s/step]INFO 2026-07-19 02:16:46 ot_train.py:646 step:35K smpl:5M ep:102K epch:20.75 loss:0.017 grdn:0.181 lr:3.7e-05 updt_s:2.343 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 194 |
+
Training: 59%|█████▉ | 35600/60000 [34:39:43<23:39:48, 3.49s/step]INFO 2026-07-19 02:28:21 ot_train.py:646 step:36K smpl:5M ep:103K epch:20.87 loss:0.017 grdn:0.166 lr:3.6e-05 updt_s:2.340 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 195 |
+
Training: 60%|█████▉ | 35800/60000 [34:51:19<23:17:48, 3.47s/step]INFO 2026-07-19 02:39:58 ot_train.py:646 step:36K smpl:5M ep:103K epch:20.98 loss:0.017 grdn:0.165 lr:3.6e-05 updt_s:2.341 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 196 |
+
Training: 60%|██████ | 36000/60000 [35:02:54<22:58:22, 3.45s/step]INFO 2026-07-19 02:51:33 ot_train.py:646 step:36K smpl:5M ep:104K epch:21.10 loss:0.017 grdn:0.167 lr:3.5e-05 updt_s:2.338 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 197 |
+
Training: 60%|██████ | 36200/60000 [35:14:31<22:57:38, 3.47s/step]INFO 2026-07-19 03:03:10 ot_train.py:646 step:36K smpl:5M ep:105K epch:21.22 loss:0.017 grdn:0.167 lr:3.5e-05 updt_s:2.333 data_s:1.147 smp/s:37 mem_gb:52.71
|
| 198 |
+
Training: 61%|██████ | 36400/60000 [35:26:08<22:43:47, 3.47s/step]INFO 2026-07-19 03:14:47 ot_train.py:646 step:36K smpl:5M ep:105K epch:21.34 loss:0.016 grdn:0.165 lr:3.4e-05 updt_s:2.341 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 199 |
+
Training: 61%|██████ | 36600/60000 [35:37:44<22:40:26, 3.49s/step]INFO 2026-07-19 03:26:23 ot_train.py:646 step:37K smpl:5M ep:106K epch:21.45 loss:0.016 grdn:0.167 lr:3.4e-05 updt_s:2.341 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 200 |
+
Training: 61%|██████▏ | 36800/60000 [35:49:21<22:22:39, 3.47s/step]INFO 2026-07-19 03:38:00 ot_train.py:646 step:37K smpl:5M ep:106K epch:21.57 loss:0.016 grdn:0.160 lr:3.3e-05 updt_s:2.343 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 201 |
+
Training: 62%|██████▏ | 37000/60000 [36:00:57<22:14:00, 3.48s/step]INFO 2026-07-19 03:49:36 ot_train.py:646 step:37K smpl:5M ep:107K epch:21.69 loss:0.016 grdn:0.161 lr:3.3e-05 updt_s:2.340 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 202 |
+
Training: 62%|██████▏ | 37200/60000 [36:12:32<22:03:42, 3.48s/step]INFO 2026-07-19 04:01:11 ot_train.py:646 step:37K smpl:5M ep:108K epch:21.81 loss:0.016 grdn:0.159 lr:3.2e-05 updt_s:2.338 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 203 |
+
Training: 62%|██████▏ | 37400/60000 [36:24:08<21:48:13, 3.47s/step]INFO 2026-07-19 04:12:47 ot_train.py:646 step:37K smpl:5M ep:108K epch:21.92 loss:0.016 grdn:0.170 lr:3.2e-05 updt_s:2.335 data_s:1.143 smp/s:37 mem_gb:52.71
|
| 204 |
+
Training: 63%|██████▎ | 37600/60000 [36:35:45<21:37:15, 3.47s/step]INFO 2026-07-19 04:24:24 ot_train.py:646 step:38K smpl:5M ep:109K epch:22.04 loss:0.015 grdn:0.169 lr:3.1e-05 updt_s:2.344 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 205 |
+
Training: 63%|██████▎ | 37800/60000 [36:47:22<21:18:32, 3.46s/step]INFO 2026-07-19 04:36:01 ot_train.py:646 step:38K smpl:5M ep:109K epch:22.16 loss:0.016 grdn:0.172 lr:3.1e-05 updt_s:2.340 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 206 |
+
Training: 63%|██████▎ | 38000/60000 [36:58:56<21:06:41, 3.45s/step]INFO 2026-07-19 04:47:35 ot_train.py:646 step:38K smpl:5M ep:110K epch:22.27 loss:0.016 grdn:0.160 lr:3.0e-05 updt_s:2.334 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 207 |
+
Training: 64%|██████▎ | 38200/60000 [37:10:32<21:00:52, 3.47s/step]INFO 2026-07-19 04:59:11 ot_train.py:646 step:38K smpl:5M ep:110K epch:22.39 loss:0.016 grdn:0.163 lr:3.0e-05 updt_s:2.339 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 208 |
+
Training: 64%|██████▍ | 38400/60000 [37:22:08<20:48:47, 3.47s/step]INFO 2026-07-19 05:10:47 ot_train.py:646 step:38K smpl:5M ep:111K epch:22.51 loss:0.015 grdn:0.168 lr:2.9e-05 updt_s:2.333 data_s:1.146 smp/s:37 mem_gb:52.71
|
| 209 |
+
Training: 64%|██████▍ | 38600/60000 [37:33:45<21:04:44, 3.55s/step]INFO 2026-07-19 05:22:24 ot_train.py:646 step:39K smpl:5M ep:112K epch:22.63 loss:0.015 grdn:0.162 lr:2.9e-05 updt_s:2.347 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 210 |
+
Training: 65%|██████▍ | 38800/60000 [37:45:24<20:52:09, 3.54s/step]INFO 2026-07-19 05:34:02 ot_train.py:646 step:39K smpl:5M ep:112K epch:22.74 loss:0.015 grdn:0.172 lr:2.8e-05 updt_s:2.350 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 211 |
+
Training: 65%|██████▌ | 39000/60000 [37:57:14<21:16:02, 3.65s/step]INFO 2026-07-19 05:45:53 ot_train.py:646 step:39K smpl:5M ep:113K epch:22.86 loss:0.015 grdn:0.159 lr:2.8e-05 updt_s:2.387 data_s:1.164 smp/s:36 mem_gb:52.79
|
| 212 |
+
Training: 65%|██████▌ | 39200/60000 [38:09:12<21:11:17, 3.67s/step]INFO 2026-07-19 05:57:50 ot_train.py:646 step:39K smpl:5M ep:113K epch:22.98 loss:0.015 grdn:0.165 lr:2.7e-05 updt_s:2.404 data_s:1.179 smp/s:36 mem_gb:52.79
|
| 213 |
+
Training: 66%|██████▌ | 39400/60000 [38:21:07<20:36:15, 3.60s/step]INFO 2026-07-19 06:09:46 ot_train.py:646 step:39K smpl:5M ep:114K epch:23.10 loss:0.014 grdn:0.163 lr:2.7e-05 updt_s:2.396 data_s:1.178 smp/s:36 mem_gb:52.79
|
| 214 |
+
Training: 66%|██████▌ | 39600/60000 [38:33:05<20:00:18, 3.53s/step]INFO 2026-07-19 06:21:44 ot_train.py:646 step:40K smpl:5M ep:114K epch:23.21 loss:0.014 grdn:0.164 lr:2.7e-05 updt_s:2.388 data_s:1.201 smp/s:36 mem_gb:52.71
|
| 215 |
+
Training: 66%|██████▋ | 39800/60000 [38:44:56<20:11:14, 3.60s/step]INFO 2026-07-19 06:33:35 ot_train.py:646 step:40K smpl:5M ep:115K epch:23.33 loss:0.013 grdn:0.156 lr:2.6e-05 updt_s:2.383 data_s:1.169 smp/s:36 mem_gb:52.79
|
| 216 |
+
Training: 67%|██████▋ | 40000/60000 [38:56:44<19:14:46, 3.46s/step]INFO 2026-07-19 06:45:22 ot_train.py:646 step:40K smpl:5M ep:116K epch:23.45 loss:0.014 grdn:0.155 lr:2.6e-05 updt_s:2.373 data_s:1.163 smp/s:36 mem_gb:52.79
|
| 217 |
+
Training: 67%|██████▋ | 40200/60000 [39:08:23<19:07:30, 3.48s/step]INFO 2026-07-19 06:57:02 ot_train.py:646 step:40K smpl:5M ep:116K epch:23.56 loss:0.013 grdn:0.156 lr:2.5e-05 updt_s:2.352 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 218 |
+
Training: 67%|██████▋ | 40400/60000 [39:20:21<19:45:19, 3.63s/step]INFO 2026-07-19 07:09:00 ot_train.py:646 step:40K smpl:5M ep:117K epch:23.68 loss:0.014 grdn:0.170 lr:2.5e-05 updt_s:2.419 data_s:1.167 smp/s:36 mem_gb:52.79
|
| 219 |
+
Training: 68%|██████▊ | 40600/60000 [39:32:22<19:50:01, 3.68s/step]INFO 2026-07-19 07:21:01 ot_train.py:646 step:41K smpl:5M ep:117K epch:23.80 loss:0.014 grdn:0.161 lr:2.4e-05 updt_s:2.431 data_s:1.173 smp/s:36 mem_gb:52.79
|
| 220 |
+
Training: 68%|██████▊ | 40800/60000 [39:44:15<19:03:35, 3.57s/step]INFO 2026-07-19 07:32:54 ot_train.py:646 step:41K smpl:5M ep:118K epch:23.92 loss:0.013 grdn:0.171 lr:2.4e-05 updt_s:2.384 data_s:1.180 smp/s:36 mem_gb:52.71
|
| 221 |
+
Training: 68%|██████▊ | 41000/60000 [39:56:10<18:39:17, 3.53s/step]INFO 2026-07-19 07:44:49 ot_train.py:646 step:41K smpl:5M ep:118K epch:24.03 loss:0.013 grdn:0.159 lr:2.3e-05 updt_s:2.393 data_s:1.178 smp/s:36 mem_gb:52.79
|
| 222 |
+
Training: 69%|██████▊ | 41200/60000 [40:08:03<19:01:12, 3.64s/step]INFO 2026-07-19 07:56:42 ot_train.py:646 step:41K smpl:5M ep:119K epch:24.15 loss:0.013 grdn:0.156 lr:2.3e-05 updt_s:2.392 data_s:1.172 smp/s:36 mem_gb:52.79
|
| 223 |
+
Training: 69%|██████▉ | 41400/60000 [40:19:44<18:01:40, 3.49s/step]INFO 2026-07-19 08:08:23 ot_train.py:646 step:41K smpl:5M ep:120K epch:24.27 loss:0.013 grdn:0.164 lr:2.2e-05 updt_s:2.356 data_s:1.147 smp/s:37 mem_gb:52.79
|
| 224 |
+
Training: 69%|██████▉ | 41600/60000 [40:31:21<17:44:00, 3.47s/step]INFO 2026-07-19 08:20:00 ot_train.py:646 step:42K smpl:5M ep:120K epch:24.38 loss:0.013 grdn:0.159 lr:2.2e-05 updt_s:2.338 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 225 |
+
Training: 70%|██████▉ | 41800/60000 [40:42:59<17:42:50, 3.50s/step]INFO 2026-07-19 08:31:38 ot_train.py:646 step:42K smpl:5M ep:121K epch:24.50 loss:0.013 grdn:0.154 lr:2.2e-05 updt_s:2.338 data_s:1.154 smp/s:37 mem_gb:52.71
|
| 226 |
+
Training: 70%|███████ | 42000/60000 [40:54:38<17:23:59, 3.48s/step]INFO 2026-07-19 08:43:17 ot_train.py:646 step:42K smpl:5M ep:121K epch:24.62 loss:0.012 grdn:0.151 lr:2.1e-05 updt_s:2.345 data_s:1.145 smp/s:37 mem_gb:52.79
|
| 227 |
+
Training: 70%|███████ | 42200/60000 [41:06:15<17:08:28, 3.47s/step]INFO 2026-07-19 08:54:54 ot_train.py:646 step:42K smpl:5M ep:122K epch:24.74 loss:0.012 grdn:0.154 lr:2.1e-05 updt_s:2.342 data_s:1.144 smp/s:37 mem_gb:52.79
|
| 228 |
+
Training: 71%|███████ | 42400/60000 [41:17:51<17:02:29, 3.49s/step]INFO 2026-07-19 09:06:30 ot_train.py:646 step:42K smpl:5M ep:123K epch:24.85 loss:0.012 grdn:0.156 lr:2.0e-05 updt_s:2.335 data_s:1.142 smp/s:37 mem_gb:52.79
|
| 229 |
+
Training: 71%|███████ | 42600/60000 [41:29:29<16:43:06, 3.46s/step]INFO 2026-07-19 09:18:08 ot_train.py:646 step:43K smpl:5M ep:123K epch:24.97 loss:0.012 grdn:0.156 lr:2.0e-05 updt_s:2.346 data_s:1.143 smp/s:37 mem_gb:52.79
|
| 230 |
+
Training: 71%|███████▏ | 42800/60000 [41:41:06<16:37:46, 3.48s/step]INFO 2026-07-19 09:29:45 ot_train.py:646 step:43K smpl:5M ep:124K epch:25.09 loss:0.012 grdn:0.152 lr:1.9e-05 updt_s:2.341 data_s:1.143 smp/s:37 mem_gb:52.79
|
| 231 |
+
Training: 72%|███████▏ | 43000/60000 [41:52:44<16:22:38, 3.47s/step]INFO 2026-07-19 09:41:23 ot_train.py:646 step:43K smpl:6M ep:124K epch:25.21 loss:0.012 grdn:0.158 lr:1.9e-05 updt_s:2.335 data_s:1.150 smp/s:37 mem_gb:52.71
|
| 232 |
+
Training: 72%|███████▏ | 43200/60000 [42:04:21<16:12:44, 3.47s/step]INFO 2026-07-19 09:53:00 ot_train.py:646 step:43K smpl:6M ep:125K epch:25.32 loss:0.012 grdn:0.163 lr:1.9e-05 updt_s:2.346 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 233 |
+
Training: 72%|███████▏ | 43400/60000 [42:15:59<16:03:56, 3.48s/step]INFO 2026-07-19 10:04:38 ot_train.py:646 step:43K smpl:6M ep:125K epch:25.44 loss:0.012 grdn:0.152 lr:1.8e-05 updt_s:2.347 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 234 |
+
Training: 73%|███████▎ | 43600/60000 [42:27:38<16:21:51, 3.59s/step]INFO 2026-07-19 10:16:17 ot_train.py:646 step:44K smpl:6M ep:126K epch:25.56 loss:0.011 grdn:0.153 lr:1.8e-05 updt_s:2.351 data_s:1.142 smp/s:37 mem_gb:52.79
|
| 235 |
+
Training: 73%|███████▎ | 43800/60000 [42:39:38<16:17:38, 3.62s/step]INFO 2026-07-19 10:28:17 ot_train.py:646 step:44K smpl:6M ep:127K epch:25.67 loss:0.012 grdn:0.157 lr:1.7e-05 updt_s:2.425 data_s:1.171 smp/s:36 mem_gb:52.79
|
| 236 |
+
Training: 73%|███████▎ | 44000/60000 [42:51:37<15:49:05, 3.56s/step]INFO 2026-07-19 10:40:16 ot_train.py:646 step:44K smpl:6M ep:127K epch:25.79 loss:0.011 grdn:0.153 lr:1.7e-05 updt_s:2.422 data_s:1.171 smp/s:36 mem_gb:52.79
|
| 237 |
+
Training: 74%|███████▎ | 44200/60000 [43:03:28<15:37:42, 3.56s/step]INFO 2026-07-19 10:52:07 ot_train.py:646 step:44K smpl:6M ep:128K epch:25.91 loss:0.011 grdn:0.155 lr:1.7e-05 updt_s:2.376 data_s:1.175 smp/s:36 mem_gb:52.71
|
| 238 |
+
Training: 74%|███████▍ | 44400/60000 [43:15:16<15:19:18, 3.54s/step]INFO 2026-07-19 11:03:55 ot_train.py:646 step:44K smpl:6M ep:128K epch:26.03 loss:0.011 grdn:0.158 lr:1.6e-05 updt_s:2.378 data_s:1.163 smp/s:36 mem_gb:52.79
|
| 239 |
+
Training: 74%|███████▍ | 44600/60000 [43:27:08<15:19:23, 3.58s/step]INFO 2026-07-19 11:15:47 ot_train.py:646 step:45K smpl:6M ep:129K epch:26.14 loss:0.011 grdn:0.156 lr:1.6e-05 updt_s:2.393 data_s:1.165 smp/s:36 mem_gb:52.79
|
| 240 |
+
Training: 75%|███████▍ | 44800/60000 [43:38:51<14:50:54, 3.52s/step]INFO 2026-07-19 11:27:30 ot_train.py:646 step:45K smpl:6M ep:129K epch:26.26 loss:0.011 grdn:0.157 lr:1.5e-05 updt_s:2.368 data_s:1.143 smp/s:36 mem_gb:52.79
|
| 241 |
+
Training: 75%|███████▌ | 45000/60000 [43:50:34<14:40:28, 3.52s/step]INFO 2026-07-19 11:39:13 ot_train.py:646 step:45K smpl:6M ep:130K epch:26.38 loss:0.011 grdn:0.152 lr:1.5e-05 updt_s:2.370 data_s:1.141 smp/s:36 mem_gb:52.79
|
| 242 |
+
Training: 75%|███████▌ | 45200/60000 [44:02:16<14:31:37, 3.53s/step]INFO 2026-07-19 11:50:55 ot_train.py:646 step:45K smpl:6M ep:131K epch:26.49 loss:0.011 grdn:0.154 lr:1.5e-05 updt_s:2.359 data_s:1.152 smp/s:36 mem_gb:52.71
|
| 243 |
+
Training: 76%|███████▌ | 45400/60000 [44:13:57<14:20:39, 3.54s/step]INFO 2026-07-19 12:02:36 ot_train.py:646 step:45K smpl:6M ep:131K epch:26.61 loss:0.011 grdn:0.149 lr:1.4e-05 updt_s:2.361 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 244 |
+
Training: 76%|███████▌ | 45600/60000 [44:25:36<14:01:37, 3.51s/step]INFO 2026-07-19 12:14:15 ot_train.py:646 step:46K smpl:6M ep:132K epch:26.73 loss:0.010 grdn:0.148 lr:1.4e-05 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 245 |
+
Training: 76%|███████▋ | 45800/60000 [44:37:16<13:53:10, 3.52s/step]INFO 2026-07-19 12:25:55 ot_train.py:646 step:46K smpl:6M ep:132K epch:26.85 loss:0.011 grdn:0.156 lr:1.4e-05 updt_s:2.359 data_s:1.141 smp/s:37 mem_gb:52.79
|
| 246 |
+
Training: 77%|███████▋ | 46000/60000 [44:48:55<13:37:09, 3.50s/step]INFO 2026-07-19 12:37:34 ot_train.py:646 step:46K smpl:6M ep:133K epch:26.96 loss:0.011 grdn:0.153 lr:1.3e-05 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 247 |
+
Training: 77%|███████▋ | 46200/60000 [45:00:35<13:21:22, 3.48s/step]INFO 2026-07-19 12:49:14 ot_train.py:646 step:46K smpl:6M ep:134K epch:27.08 loss:0.011 grdn:0.163 lr:1.3e-05 updt_s:2.355 data_s:1.142 smp/s:37 mem_gb:52.79
|
| 248 |
+
Training: 77%|███████▋ | 46400/60000 [45:12:15<13:07:43, 3.48s/step]INFO 2026-07-19 13:00:53 ot_train.py:646 step:46K smpl:6M ep:134K epch:27.20 loss:0.010 grdn:0.149 lr:1.3e-05 updt_s:2.345 data_s:1.151 smp/s:37 mem_gb:52.71
|
| 249 |
+
Training: 78%|███████▊ | 46600/60000 [45:23:54<13:01:30, 3.50s/step]INFO 2026-07-19 13:12:33 ot_train.py:646 step:47K smpl:6M ep:135K epch:27.32 loss:0.010 grdn:0.153 lr:1.2e-05 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 250 |
+
Training: 78%|███████▊ | 46800/60000 [45:35:33<12:45:56, 3.48s/step]INFO 2026-07-19 13:24:12 ot_train.py:646 step:47K smpl:6M ep:135K epch:27.43 loss:0.010 grdn:0.146 lr:1.2e-05 updt_s:2.358 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 251 |
+
Training: 78%|███████▊ | 47000/60000 [45:47:14<12:36:07, 3.49s/step]INFO 2026-07-19 13:35:53 ot_train.py:646 step:47K smpl:6M ep:136K epch:27.55 loss:0.010 grdn:0.154 lr:1.1e-05 updt_s:2.364 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 252 |
+
Training: 79%|███████▊ | 47200/60000 [45:58:52<12:20:29, 3.47s/step]INFO 2026-07-19 13:47:31 ot_train.py:646 step:47K smpl:6M ep:136K epch:27.67 loss:0.010 grdn:0.146 lr:1.1e-05 updt_s:2.357 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 253 |
+
Training: 79%|███████▉ | 47400/60000 [46:10:31<12:10:46, 3.48s/step]INFO 2026-07-19 13:59:10 ot_train.py:646 step:47K smpl:6M ep:137K epch:27.78 loss:0.010 grdn:0.145 lr:1.1e-05 updt_s:2.355 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 254 |
+
Training: 79%|███████▉ | 47600/60000 [46:22:09<11:59:36, 3.48s/step]INFO 2026-07-19 14:10:48 ot_train.py:646 step:48K smpl:6M ep:138K epch:27.90 loss:0.010 grdn:0.149 lr:1.0e-05 updt_s:2.345 data_s:1.144 smp/s:37 mem_gb:52.71
|
| 255 |
+
Training: 80%|███████▉ | 47800/60000 [46:33:48<11:44:59, 3.47s/step]INFO 2026-07-19 14:22:27 ot_train.py:646 step:48K smpl:6M ep:138K epch:28.02 loss:0.009 grdn:0.150 lr:1.0e-05 updt_s:2.358 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 256 |
+
Training: 80%|████████ | 48000/60000 [46:45:27<11:40:44, 3.50s/step]INFO 2026-07-19 14:34:06 ot_train.py:646 step:48K smpl:6M ep:139K epch:28.14 loss:0.009 grdn:0.153 lr:9.9e-06 updt_s:2.359 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 257 |
+
Training: 80%|████████ | 48200/60000 [46:57:06<11:25:32, 3.49s/step]INFO 2026-07-19 14:45:45 ot_train.py:646 step:48K smpl:6M ep:139K epch:28.25 loss:0.009 grdn:0.153 lr:9.5e-06 updt_s:2.359 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 258 |
+
Training: 81%|████████ | 48400/60000 [47:08:46<11:17:40, 3.51s/step]INFO 2026-07-19 14:57:25 ot_train.py:646 step:48K smpl:6M ep:140K epch:28.37 loss:0.009 grdn:0.152 lr:9.2e-06 updt_s:2.357 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 259 |
+
Training: 81%|████████ | 48600/60000 [47:20:27<11:04:46, 3.50s/step]INFO 2026-07-19 15:09:06 ot_train.py:646 step:49K smpl:6M ep:140K epch:28.49 loss:0.009 grdn:0.152 lr:8.9e-06 updt_s:2.361 data_s:1.145 smp/s:37 mem_gb:52.71
|
| 260 |
+
Training: 81%|████████▏ | 48800/60000 [47:32:08<10:59:00, 3.53s/step]INFO 2026-07-19 15:20:46 ot_train.py:646 step:49K smpl:6M ep:141K epch:28.61 loss:0.009 grdn:0.141 lr:8.6e-06 updt_s:2.365 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 261 |
+
Training: 82%|████████▏ | 49000/60000 [47:43:46<10:45:43, 3.52s/step]INFO 2026-07-19 15:32:25 ot_train.py:646 step:49K smpl:6M ep:142K epch:28.72 loss:0.009 grdn:0.149 lr:8.3e-06 updt_s:2.358 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 262 |
+
Training: 82%|████████▏ | 49200/60000 [47:55:25<10:31:36, 3.51s/step]INFO 2026-07-19 15:44:04 ot_train.py:646 step:49K smpl:6M ep:142K epch:28.84 loss:0.009 grdn:0.147 lr:8.1e-06 updt_s:2.360 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 263 |
+
Training: 82%|████████▏ | 49400/60000 [48:07:06<10:20:20, 3.51s/step]INFO 2026-07-19 15:55:44 ot_train.py:646 step:49K smpl:6M ep:143K epch:28.96 loss:0.009 grdn:0.144 lr:7.8e-06 updt_s:2.362 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 264 |
+
Training: 83%|████████▎ | 49600/60000 [48:18:44<10:08:32, 3.51s/step]INFO 2026-07-19 16:07:22 ot_train.py:646 step:50K smpl:6M ep:143K epch:29.07 loss:0.009 grdn:0.144 lr:7.5e-06 updt_s:2.351 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 265 |
+
Training: 83%|████████▎ | 49800/60000 [48:30:22<9:51:39, 3.48s/step]INFO 2026-07-19 16:19:01 ot_train.py:646 step:50K smpl:6M ep:144K epch:29.19 loss:0.009 grdn:0.147 lr:7.2e-06 updt_s:2.346 data_s:1.144 smp/s:37 mem_gb:52.71
|
| 266 |
+
Training: 83%|████████▎ | 50000/60000 [48:42:01<9:41:47, 3.49s/step]INFO 2026-07-19 16:30:40 ot_train.py:646 step:50K smpl:6M ep:144K epch:29.31 loss:0.009 grdn:0.150 lr:6.9e-06 updt_s:2.355 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 267 |
+
Training: 84%|████████▎ | 50200/60000 [48:53:40<9:33:55, 3.51s/step]INFO 2026-07-19 16:42:19 ot_train.py:646 step:50K smpl:6M ep:145K epch:29.43 loss:0.008 grdn:0.142 lr:6.7e-06 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 268 |
+
Training: 84%|████████▍ | 50400/60000 [49:05:19<9:17:31, 3.48s/step]INFO 2026-07-19 16:53:57 ot_train.py:646 step:50K smpl:6M ep:146K epch:29.54 loss:0.008 grdn:0.147 lr:6.4e-06 updt_s:2.354 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 269 |
+
Training: 84%|████████▍ | 50600/60000 [49:16:58<9:07:37, 3.50s/step]INFO 2026-07-19 17:05:37 ot_train.py:646 step:51K smpl:6M ep:146K epch:29.66 loss:0.009 grdn:0.156 lr:6.2e-06 updt_s:2.356 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 270 |
+
Training: 85%|████████▍ | 50800/60000 [49:28:36<8:52:51, 3.48s/step]INFO 2026-07-19 17:17:15 ot_train.py:646 step:51K smpl:7M ep:147K epch:29.78 loss:0.008 grdn:0.142 lr:5.9e-06 updt_s:2.353 data_s:1.138 smp/s:37 mem_gb:52.79
|
| 271 |
+
Training: 85%|████████▌ | 51000/60000 [49:40:17<8:42:42, 3.48s/step]INFO 2026-07-19 17:28:56 ot_train.py:646 step:51K smpl:7M ep:147K epch:29.89 loss:0.009 grdn:0.152 lr:5.7e-06 updt_s:2.352 data_s:1.149 smp/s:37 mem_gb:52.71
|
| 272 |
+
Training: 85%|████████▌ | 51200/60000 [49:51:56<8:35:08, 3.51s/step]INFO 2026-07-19 17:40:35 ot_train.py:646 step:51K smpl:7M ep:148K epch:30.01 loss:0.008 grdn:0.145 lr:5.4e-06 updt_s:2.356 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 273 |
+
Training: 86%|████████▌ | 51400/60000 [50:03:35<8:23:28, 3.51s/step]INFO 2026-07-19 17:52:14 ot_train.py:646 step:51K smpl:7M ep:149K epch:30.13 loss:0.008 grdn:0.145 lr:5.2e-06 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 274 |
+
Training: 86%|████████▌ | 51600/60000 [50:15:14<8:14:26, 3.53s/step]INFO 2026-07-19 18:03:53 ot_train.py:646 step:52K smpl:7M ep:149K epch:30.25 loss:0.008 grdn:0.145 lr:5.0e-06 updt_s:2.353 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 275 |
+
Training: 86%|████████▋ | 51800/60000 [50:26:52<7:56:31, 3.49s/step]INFO 2026-07-19 18:15:31 ot_train.py:646 step:52K smpl:7M ep:150K epch:30.36 loss:0.008 grdn:0.143 lr:4.7e-06 updt_s:2.353 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 276 |
+
Training: 87%|████████▋ | 52000/60000 [50:38:31<7:44:17, 3.48s/step]INFO 2026-07-19 18:27:10 ot_train.py:646 step:52K smpl:7M ep:150K epch:30.48 loss:0.008 grdn:0.145 lr:4.5e-06 updt_s:2.352 data_s:1.144 smp/s:37 mem_gb:52.71
|
| 277 |
+
Training: 87%|████████▋ | 52200/60000 [50:50:10<7:31:46, 3.48s/step]INFO 2026-07-19 18:38:49 ot_train.py:646 step:52K smpl:7M ep:151K epch:30.60 loss:0.008 grdn:0.141 lr:4.3e-06 updt_s:2.357 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 278 |
+
Training: 87%|████████▋ | 52400/60000 [51:01:48<7:22:32, 3.49s/step]INFO 2026-07-19 18:50:27 ot_train.py:646 step:52K smpl:7M ep:151K epch:30.72 loss:0.008 grdn:0.140 lr:4.1e-06 updt_s:2.355 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 279 |
+
Training: 88%|████████▊ | 52600/60000 [51:13:26<7:13:34, 3.52s/step]INFO 2026-07-19 19:02:05 ot_train.py:646 step:53K smpl:7M ep:152K epch:30.83 loss:0.008 grdn:0.143 lr:3.9e-06 updt_s:2.357 data_s:1.131 smp/s:37 mem_gb:52.79
|
| 280 |
+
Training: 88%|████████▊ | 52800/60000 [51:25:05<6:58:35, 3.49s/step]INFO 2026-07-19 19:13:44 ot_train.py:646 step:53K smpl:7M ep:153K epch:30.95 loss:0.008 grdn:0.143 lr:3.7e-06 updt_s:2.357 data_s:1.137 smp/s:37 mem_gb:52.79
|
| 281 |
+
Training: 88%|████████▊ | 53000/60000 [51:36:46<6:47:17, 3.49s/step]INFO 2026-07-19 19:25:25 ot_train.py:646 step:53K smpl:7M ep:153K epch:31.07 loss:0.008 grdn:0.142 lr:3.5e-06 updt_s:2.364 data_s:1.139 smp/s:37 mem_gb:52.79
|
| 282 |
+
Training: 89%|████████▊ | 53200/60000 [51:48:28<6:34:46, 3.48s/step]INFO 2026-07-19 19:37:07 ot_train.py:646 step:53K smpl:7M ep:154K epch:31.18 loss:0.008 grdn:0.146 lr:3.3e-06 updt_s:2.349 data_s:1.157 smp/s:37 mem_gb:52.71
|
| 283 |
+
Training: 89%|████████▉ | 53400/60000 [52:00:10<6:24:41, 3.50s/step]INFO 2026-07-19 19:48:49 ot_train.py:646 step:53K smpl:7M ep:154K epch:31.30 loss:0.008 grdn:0.144 lr:3.1e-06 updt_s:2.358 data_s:1.150 smp/s:36 mem_gb:52.79
|
| 284 |
+
Training: 89%|████████▉ | 53600/60000 [52:11:49<6:11:16, 3.48s/step]INFO 2026-07-19 20:00:28 ot_train.py:646 step:54K smpl:7M ep:155K epch:31.42 loss:0.008 grdn:0.142 lr:2.9e-06 updt_s:2.353 data_s:1.140 smp/s:37 mem_gb:52.79
|
| 285 |
+
Training: 90%|████████▉ | 53800/60000 [52:23:27<5:59:18, 3.48s/step]INFO 2026-07-19 20:12:06 ot_train.py:646 step:54K smpl:7M ep:155K epch:31.54 loss:0.007 grdn:0.147 lr:2.7e-06 updt_s:2.356 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 286 |
+
Training: 90%|█████████ | 54000/60000 [52:35:08<5:49:45, 3.50s/step]INFO 2026-07-19 20:23:47 ot_train.py:646 step:54K smpl:7M ep:156K epch:31.65 loss:0.008 grdn:0.140 lr:2.6e-06 updt_s:2.365 data_s:1.135 smp/s:37 mem_gb:52.79
|
| 287 |
+
Training: 90%|█████████ | 54200/60000 [52:46:46<5:42:39, 3.54s/step]INFO 2026-07-19 20:35:25 ot_train.py:646 step:54K smpl:7M ep:157K epch:31.77 loss:0.008 grdn:0.141 lr:2.4e-06 updt_s:2.348 data_s:1.141 smp/s:37 mem_gb:52.71
|
| 288 |
+
Training: 91%|█████████ | 54400/60000 [52:58:25<5:23:29, 3.47s/step]INFO 2026-07-19 20:47:04 ot_train.py:646 step:54K smpl:7M ep:157K epch:31.89 loss:0.007 grdn:0.140 lr:2.2e-06 updt_s:2.359 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 289 |
+
Training: 91%|█████████ | 54600/60000 [53:10:04<5:14:13, 3.49s/step]INFO 2026-07-19 20:58:43 ot_train.py:646 step:55K smpl:7M ep:158K epch:32.00 loss:0.008 grdn:0.148 lr:2.1e-06 updt_s:2.360 data_s:1.134 smp/s:37 mem_gb:52.79
|
| 290 |
+
Training: 91%|█████████▏| 54800/60000 [53:21:43<5:02:36, 3.49s/step]INFO 2026-07-19 21:10:22 ot_train.py:646 step:55K smpl:7M ep:158K epch:32.12 loss:0.008 grdn:0.147 lr:1.9e-06 updt_s:2.358 data_s:1.136 smp/s:37 mem_gb:52.79
|
| 291 |
+
Training: 92%|█████████▏| 55000/60000 [53:33:21<4:52:34, 3.51s/step]INFO 2026-07-19 21:22:00 ot_train.py:646 step:55K smpl:7M ep:159K epch:32.24 loss:0.007 grdn:0.146 lr:1.8e-06 updt_s:2.353 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 292 |
+
Training: 92%|█████████▏| 55200/60000 [53:44:58<4:39:57, 3.50s/step]INFO 2026-07-19 21:33:37 ot_train.py:646 step:55K smpl:7M ep:160K epch:32.36 loss:0.007 grdn:0.141 lr:1.7e-06 updt_s:2.352 data_s:1.131 smp/s:37 mem_gb:52.79
|
| 293 |
+
Training: 92%|█████████▏| 55400/60000 [53:56:35<4:26:14, 3.47s/step]INFO 2026-07-19 21:45:14 ot_train.py:646 step:55K smpl:7M ep:160K epch:32.47 loss:0.007 grdn:0.138 lr:1.5e-06 updt_s:2.345 data_s:1.139 smp/s:37 mem_gb:52.71
|
| 294 |
+
Training: 93%|█████████▎| 55600/60000 [54:08:13<4:17:21, 3.51s/step]INFO 2026-07-19 21:56:51 ot_train.py:646 step:56K smpl:7M ep:161K epch:32.59 loss:0.007 grdn:0.143 lr:1.4e-06 updt_s:2.357 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 295 |
+
Training: 93%|█████████▎| 55800/60000 [54:19:50<4:03:56, 3.48s/step]INFO 2026-07-19 22:08:29 ot_train.py:646 step:56K smpl:7M ep:161K epch:32.71 loss:0.007 grdn:0.150 lr:1.3e-06 updt_s:2.356 data_s:1.129 smp/s:37 mem_gb:52.79
|
| 296 |
+
Training: 93%|█████████▎| 56000/60000 [54:31:28<3:51:33, 3.47s/step]INFO 2026-07-19 22:20:07 ot_train.py:646 step:56K smpl:7M ep:162K epch:32.83 loss:0.007 grdn:0.140 lr:1.2e-06 updt_s:2.358 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 297 |
+
Training: 94%|█████████▎| 56200/60000 [54:43:06<3:40:39, 3.48s/step]INFO 2026-07-19 22:31:44 ot_train.py:646 step:56K smpl:7M ep:162K epch:32.94 loss:0.007 grdn:0.142 lr:1.1e-06 updt_s:2.356 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 298 |
+
Training: 94%|█████████▍| 56400/60000 [54:54:44<3:27:34, 3.46s/step]INFO 2026-07-19 22:43:23 ot_train.py:646 step:56K smpl:7M ep:163K epch:33.06 loss:0.007 grdn:0.144 lr:9.5e-07 updt_s:2.360 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 299 |
+
Training: 94%|█████████▍| 56600/60000 [55:06:21<3:17:27, 3.48s/step]INFO 2026-07-19 22:55:00 ot_train.py:646 step:57K smpl:7M ep:164K epch:33.18 loss:0.007 grdn:0.139 lr:8.5e-07 updt_s:2.349 data_s:1.138 smp/s:37 mem_gb:52.71
|
| 300 |
+
Training: 95%|█████████▍| 56800/60000 [55:17:59<3:06:16, 3.49s/step]INFO 2026-07-19 23:06:38 ot_train.py:646 step:57K smpl:7M ep:164K epch:33.29 loss:0.007 grdn:0.143 lr:7.6e-07 updt_s:2.356 data_s:1.128 smp/s:37 mem_gb:52.79
|
| 301 |
+
Training: 95%|█████████▌| 57000/60000 [55:29:36<2:54:47, 3.50s/step]INFO 2026-07-19 23:18:15 ot_train.py:646 step:57K smpl:7M ep:165K epch:33.41 loss:0.007 grdn:0.140 lr:6.7e-07 updt_s:2.356 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 302 |
+
Training: 95%|█████████▌| 57200/60000 [55:41:14<2:42:30, 3.48s/step]INFO 2026-07-19 23:29:53 ot_train.py:646 step:57K smpl:7M ep:165K epch:33.53 loss:0.007 grdn:0.143 lr:5.9e-07 updt_s:2.356 data_s:1.132 smp/s:37 mem_gb:52.79
|
| 303 |
+
Training: 96%|█████████▌| 57400/60000 [55:52:52<2:31:23, 3.49s/step]INFO 2026-07-19 23:41:31 ot_train.py:646 step:57K smpl:7M ep:166K epch:33.65 loss:0.007 grdn:0.142 lr:5.1e-07 updt_s:2.358 data_s:1.131 smp/s:37 mem_gb:52.79
|
| 304 |
+
Training: 96%|█████████▌| 57600/60000 [56:04:31<2:20:02, 3.50s/step]INFO 2026-07-19 23:53:10 ot_train.py:646 step:58K smpl:7M ep:166K epch:33.76 loss:0.007 grdn:0.142 lr:4.4e-07 updt_s:2.352 data_s:1.142 smp/s:37 mem_gb:52.71
|
| 305 |
+
Training: 96%|█████████▋| 57800/60000 [56:16:10<2:07:43, 3.48s/step]INFO 2026-07-20 00:04:49 ot_train.py:646 step:58K smpl:7M ep:167K epch:33.88 loss:0.007 grdn:0.142 lr:3.7e-07 updt_s:2.362 data_s:1.131 smp/s:37 mem_gb:52.79
|
| 306 |
+
Training: 97%|█████████▋| 58000/60000 [56:27:49<1:56:15, 3.49s/step]INFO 2026-07-20 00:16:28 ot_train.py:646 step:58K smpl:7M ep:168K epch:34.00 loss:0.007 grdn:0.139 lr:3.1e-07 updt_s:2.361 data_s:1.132 smp/s:37 mem_gb:52.79
|
| 307 |
+
Training: 97%|█████████▋| 58200/60000 [56:39:28<1:46:44, 3.56s/step]INFO 2026-07-20 00:28:06 ot_train.py:646 step:58K smpl:7M ep:168K epch:34.12 loss:0.007 grdn:0.133 lr:2.5e-07 updt_s:2.360 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 308 |
+
Training: 97%|█████████▋| 58400/60000 [56:51:05<1:32:58, 3.49s/step]INFO 2026-07-20 00:39:44 ot_train.py:646 step:58K smpl:7M ep:169K epch:34.23 loss:0.007 grdn:0.144 lr:2.0e-07 updt_s:2.358 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 309 |
+
Training: 98%|█████████▊| 58600/60000 [57:02:44<1:21:35, 3.50s/step]INFO 2026-07-20 00:51:23 ot_train.py:646 step:59K smpl:8M ep:169K epch:34.35 loss:0.007 grdn:0.140 lr:1.6e-07 updt_s:2.360 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 310 |
+
Training: 98%|█████████▊| 58800/60000 [57:14:23<1:09:39, 3.48s/step]INFO 2026-07-20 01:03:02 ot_train.py:646 step:59K smpl:8M ep:170K epch:34.47 loss:0.007 grdn:0.143 lr:1.2e-07 updt_s:2.351 data_s:1.139 smp/s:37 mem_gb:52.71
|
| 311 |
+
Training: 98%|█████████▊| 59000/60000 [57:26:01<58:25, 3.51s/step]INFO 2026-07-20 01:14:40 ot_train.py:646 step:59K smpl:8M ep:170K epch:34.58 loss:0.007 grdn:0.142 lr:8.4e-08 updt_s:2.360 data_s:1.131 smp/s:37 mem_gb:52.79
|
| 312 |
+
Training: 99%|█████████▊| 59200/60000 [57:37:41<47:08, 3.54s/step]INFO 2026-07-20 01:26:20 ot_train.py:646 step:59K smpl:8M ep:171K epch:34.70 loss:0.007 grdn:0.144 lr:5.7e-08 updt_s:2.364 data_s:1.133 smp/s:37 mem_gb:52.79
|
| 313 |
+
Training: 99%|█████████▉| 59400/60000 [57:49:20<34:51, 3.49s/step]INFO 2026-07-20 01:37:59 ot_train.py:646 step:59K smpl:8M ep:172K epch:34.82 loss:0.007 grdn:0.140 lr:3.4e-08 updt_s:2.361 data_s:1.130 smp/s:37 mem_gb:52.79
|
| 314 |
+
Training: 99%|█████████▉| 59600/60000 [58:00:58<23:15, 3.49s/step]INFO 2026-07-20 01:49:37 ot_train.py:646 step:60K smpl:8M ep:172K epch:34.94 loss:0.007 grdn:0.137 lr:1.8e-08 updt_s:2.361 data_s:1.129 smp/s:37 mem_gb:52.79
|
| 315 |
+
Training: 100%|█████████▉| 59800/60000 [58:12:35<11:36, 3.48s/step]INFO 2026-07-20 02:01:14 ot_train.py:646 step:60K smpl:8M ep:173K epch:35.05 loss:0.007 grdn:0.139 lr:6.5e-09 updt_s:2.355 data_s:1.129 smp/s:37 mem_gb:52.79
|
| 316 |
+
Training: 100%|██████████| 60000/60000 [58:24:13<00:00, 3.51s/step]INFO 2026-07-20 02:12:52 ot_train.py:646 step:60K smpl:8M ep:173K epch:35.17 loss:0.007 grdn:0.147 lr:9.2e-10 updt_s:2.349 data_s:1.139 smp/s:37 mem_gb:52.71
|
| 317 |
+
INFO 2026-07-20 02:12:52 ot_train.py:691 Checkpoint policy after step 60000
|
| 318 |
+
INFO 2026-07-20 02:13:16 in_yaml.py:1245 Saved config, action-space, prompt, and phase provenance in /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline and /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/checkpoints/060000
|
| 319 |
+
Training: 100%|██████████| 60000/60000 [58:24:37<00:00, 3.50s/step]
|
| 320 |
+
INFO 2026-07-20 02:13:16 ot_train.py:777 End of training
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/requirements.txt
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
multiprocess==0.70.19
|
| 2 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 3 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 4 |
+
nvidia-nvjitlink==13.0.88
|
| 5 |
+
numpy==2.2.6
|
| 6 |
+
shellingham==1.5.4
|
| 7 |
+
pip==26.1.2
|
| 8 |
+
zipp==4.1.0
|
| 9 |
+
mypy_extensions==1.1.0
|
| 10 |
+
pillow==12.3.0
|
| 11 |
+
wheel==0.47.0
|
| 12 |
+
cuda-pathfinder==1.5.6
|
| 13 |
+
dm-tree==0.1.10
|
| 14 |
+
hf-xet==1.5.1
|
| 15 |
+
packaging==25.0
|
| 16 |
+
typing-inspect==0.9.0
|
| 17 |
+
nvidia-cuda-runtime==13.0.96
|
| 18 |
+
pyyaml-include==1.4.1
|
| 19 |
+
wrapt==2.2.2
|
| 20 |
+
pytz==2026.2
|
| 21 |
+
Pygments==2.20.0
|
| 22 |
+
urllib3==2.7.0
|
| 23 |
+
yarl==1.24.2
|
| 24 |
+
regex==2026.7.10
|
| 25 |
+
anyio==4.14.2
|
| 26 |
+
timm==1.0.28
|
| 27 |
+
typer==0.26.8
|
| 28 |
+
tokenizers==0.22.2
|
| 29 |
+
lerobot==0.6.0
|
| 30 |
+
annotated-doc==0.0.4
|
| 31 |
+
importlib_metadata==9.0.0
|
| 32 |
+
idna==3.18
|
| 33 |
+
mpmath==1.3.0
|
| 34 |
+
platformdirs==4.10.0
|
| 35 |
+
decord==0.6.0
|
| 36 |
+
transformers==5.5.4
|
| 37 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 38 |
+
accelerate==1.14.0
|
| 39 |
+
torchvision==0.26.0
|
| 40 |
+
torchcodec==0.11.1
|
| 41 |
+
nvidia-cufft==12.0.0.61
|
| 42 |
+
mdurl==0.1.2
|
| 43 |
+
sentry-sdk==2.64.0
|
| 44 |
+
av==15.1.0
|
| 45 |
+
nvidia-cusparse==12.6.3.3
|
| 46 |
+
PyYAML==6.0.3
|
| 47 |
+
python-dateutil==2.9.0.post0
|
| 48 |
+
protobuf==7.35.1
|
| 49 |
+
cmake==4.1.3
|
| 50 |
+
pydantic_core==2.46.4
|
| 51 |
+
requests==2.34.2
|
| 52 |
+
typing-inspection==0.4.2
|
| 53 |
+
nvidia-cufile==1.15.1.6
|
| 54 |
+
MarkupSafe==3.0.3
|
| 55 |
+
wandb==0.27.2
|
| 56 |
+
smmap==5.0.3
|
| 57 |
+
nvidia-cuda-cupti==13.0.85
|
| 58 |
+
charset-normalizer==3.4.9
|
| 59 |
+
h11==0.16.0
|
| 60 |
+
mergedeep==1.3.4
|
| 61 |
+
opencv-python-headless==4.13.0.92
|
| 62 |
+
draccus==0.10.0
|
| 63 |
+
annotated-types==0.7.0
|
| 64 |
+
rich==15.0.0
|
| 65 |
+
networkx==3.6.1
|
| 66 |
+
GitPython==3.1.51
|
| 67 |
+
safetensors==0.8.0
|
| 68 |
+
markdown-it-py==4.2.0
|
| 69 |
+
gitdb==4.0.12
|
| 70 |
+
toml==0.10.2
|
| 71 |
+
filelock==3.29.7
|
| 72 |
+
diffusers==0.35.2
|
| 73 |
+
typing_extensions==4.16.0
|
| 74 |
+
certifi==2026.6.17
|
| 75 |
+
tzdata==2026.3
|
| 76 |
+
nvidia-nccl-cu13==2.28.9
|
| 77 |
+
aiohappyeyeballs==2.7.1
|
| 78 |
+
setuptools==80.10.2
|
| 79 |
+
sympy==1.14.0
|
| 80 |
+
torch==2.11.0
|
| 81 |
+
einops==0.8.2
|
| 82 |
+
nvidia-curand==10.4.0.35
|
| 83 |
+
httpx==0.28.1
|
| 84 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 85 |
+
frozenlist==1.8.0
|
| 86 |
+
termcolor==3.3.0
|
| 87 |
+
multidict==6.7.1
|
| 88 |
+
httpcore==1.0.9
|
| 89 |
+
psutil==7.2.2
|
| 90 |
+
fsspec==2026.2.0
|
| 91 |
+
dill==0.4.1
|
| 92 |
+
pydantic==2.13.4
|
| 93 |
+
cuda-bindings==13.3.1
|
| 94 |
+
cloudpickle==3.1.2
|
| 95 |
+
aiohttp==3.14.1
|
| 96 |
+
datasets==4.8.5
|
| 97 |
+
gymnasium==1.3.0
|
| 98 |
+
jsonlines==4.0.0
|
| 99 |
+
pyarrow==25.0.0
|
| 100 |
+
attrs==26.1.0
|
| 101 |
+
peft==0.19.1
|
| 102 |
+
nvidia-cublas==13.1.0.3
|
| 103 |
+
propcache==0.5.2
|
| 104 |
+
Jinja2==3.1.6
|
| 105 |
+
aiosignal==1.4.0
|
| 106 |
+
nvidia-nvtx==13.0.85
|
| 107 |
+
triton==3.6.0
|
| 108 |
+
tqdm==4.68.4
|
| 109 |
+
pandas==2.3.3
|
| 110 |
+
click==8.4.2
|
| 111 |
+
Farama-Notifications==0.0.6
|
| 112 |
+
six==1.17.0
|
| 113 |
+
huggingface_hub==1.23.0
|
| 114 |
+
nvidia-cusolver==12.0.4.66
|
| 115 |
+
cuda-toolkit==13.0.2
|
| 116 |
+
xxhash==3.8.1
|
| 117 |
+
absl-py==2.5.0
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-110-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.12.13",
|
| 4 |
+
"startedAt": "2026-07-17T06:48:30.892778Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config_path=/tmp/insight_lerobot_cfg_s1elc9y_/baseline.yaml",
|
| 7 |
+
"--batch_size=128"
|
| 8 |
+
],
|
| 9 |
+
"program": "/home/seonho/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
|
| 10 |
+
"codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py",
|
| 11 |
+
"codePathLocal": "my_scripts/lerobot_train_yaml.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "git@github.com:paragon7060/clvla.git",
|
| 14 |
+
"commit": "a970d9f98f189faccce13cf2fb61ccf5d55921f3"
|
| 15 |
+
},
|
| 16 |
+
"root": "/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline",
|
| 17 |
+
"host": "worker1",
|
| 18 |
+
"executable": "/home/seonho/miniconda3/envs/lerobot060_groot/bin/python",
|
| 19 |
+
"cpu_count": 32,
|
| 20 |
+
"cpu_count_logical": 64,
|
| 21 |
+
"gpu": "NVIDIA A100 80GB PCIe",
|
| 22 |
+
"gpu_count": 4,
|
| 23 |
+
"disk": {
|
| 24 |
+
"/": {
|
| 25 |
+
"total": "3778130485248",
|
| 26 |
+
"used": "3148893519872"
|
| 27 |
+
}
|
| 28 |
+
},
|
| 29 |
+
"memory": {
|
| 30 |
+
"total": "1081620799488"
|
| 31 |
+
},
|
| 32 |
+
"gpu_nvidia": [
|
| 33 |
+
{
|
| 34 |
+
"name": "NVIDIA A100 80GB PCIe",
|
| 35 |
+
"memoryTotal": "85899345920",
|
| 36 |
+
"cudaCores": 6912,
|
| 37 |
+
"architecture": "Ampere",
|
| 38 |
+
"uuid": "GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"name": "NVIDIA A100 80GB PCIe",
|
| 42 |
+
"memoryTotal": "85899345920",
|
| 43 |
+
"cudaCores": 6912,
|
| 44 |
+
"architecture": "Ampere",
|
| 45 |
+
"uuid": "GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"name": "NVIDIA A100 80GB PCIe",
|
| 49 |
+
"memoryTotal": "85899345920",
|
| 50 |
+
"cudaCores": 6912,
|
| 51 |
+
"architecture": "Ampere",
|
| 52 |
+
"uuid": "GPU-ca1c0319-e178-a3c3-ac41-386c2723deac"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"name": "NVIDIA A100 80GB PCIe",
|
| 56 |
+
"memoryTotal": "85899345920",
|
| 57 |
+
"cudaCores": 6912,
|
| 58 |
+
"architecture": "Ampere",
|
| 59 |
+
"uuid": "GPU-47b6036e-4fde-e399-5515-f73eed752c6d"
|
| 60 |
+
}
|
| 61 |
+
],
|
| 62 |
+
"cudaVersion": "13.2",
|
| 63 |
+
"writerId": "pbf296fi8oqpaqrnve886ppcvwjoxaoj"
|
| 64 |
+
}
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"train/lr":9.22312585679652e-10,"train/samples_per_s":36.69309955325707,"train/samples":7680000,"train/gpu_mem_gb":52.70832591533661,"train/grad_norm":0.14739508057013154,"_timestamp":1.7844811724342055e+09,"_wandb":{"runtime":210284},"train/steps":60000,"train/episodes":173388.83622525382,"_runtime":210284,"train/epochs":35.17014933575128,"train/update_s":2.3494585001049564,"_step":60000,"train/dataloading_s":1.1389358181296847,"train/loss":0.009968766942620277}
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-core.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-17T15:48:30.925035413+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpldv3yhws/port-3896698.txt","pid":3896698,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-07-17T15:48:30.925459259+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3896698}
|
| 3 |
+
{"time":"2026-07-17T15:48:30.925437749+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3896698-3896942-2058348239/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-07-17T15:48:31.110302565+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-07-17T15:48:31.116186898+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"a3ogbmg4","id":"1(@)"}
|
| 6 |
+
{"time":"2026-07-17T15:48:31.564584695+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"a3ogbmg4","id":"1(@)"}
|
| 7 |
+
{"time":"2026-07-17T15:48:37.272889118+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ijhibshzn6ev"}
|
| 8 |
+
{"time":"2026-07-20T02:13:16.832444619+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-07-20T02:13:16.832520908+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-07-20T02:13:16.832554047+09:00","level":"INFO","msg":"server is shutting down"}
|
| 11 |
+
{"time":"2026-07-20T02:13:16.832569444+09:00","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
|
| 12 |
+
{"time":"2026-07-20T02:13:16.832658051+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 13 |
+
{"time":"2026-07-20T02:13:16.83267221+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3896698-3896942-2058348239/socket","Net":"unix"}}
|
| 14 |
+
{"time":"2026-07-20T02:13:18.668035591+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 15 |
+
{"time":"2026-07-20T02:13:18.668080494+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 16 |
+
{"time":"2026-07-20T02:13:18.668095693+09:00","level":"INFO","msg":"server is closed"}
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
|
| 2 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Configure stats pid to 3896698
|
| 3 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-17 15:48:30,893 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug.log
|
| 5 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/logs/debug-internal.log
|
| 6 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():766] calling init triggers
|
| 7 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/seonho/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/seonho/groot_insight/train_ckpt/Abs_Joint/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 44, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}}
|
| 9 |
+
2026-07-17 15:48:30,894 INFO MainThread:3896698 [wandb_init.py:init():814] starting backend
|
| 10 |
+
2026-07-17 15:48:31,110 INFO MainThread:3896698 [wandb_init.py:init():829] sending inform_init request
|
| 11 |
+
2026-07-17 15:48:31,565 INFO MainThread:3896698 [wandb_init.py:init():834] backend started and connected
|
| 12 |
+
2026-07-17 15:48:31,568 INFO MainThread:3896698 [wandb_init.py:init():904] updated telemetry
|
| 13 |
+
2026-07-17 15:48:31,579 INFO MainThread:3896698 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-17 15:48:32,131 INFO MainThread:3896698 [wandb_init.py:init():972] starting run threads in backend
|
| 15 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_console_start():2523] atexit reg
|
| 16 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-07-17 15:48:32,264 INFO MainThread:3896698 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-07-17 15:48:32,267 INFO MainThread:3896698 [wandb_init.py:init():1010] run started, returning control to user process
|
| 20 |
+
2026-07-20 02:13:16,832 INFO wandb-AsyncioManager-main:3896698 [service_client.py:_forward_responses():122] Reached EOF.
|
| 21 |
+
2026-07-20 02:13:16,833 INFO wandb-AsyncioManager-main:3896698 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
|
| 22 |
+
2026-07-20 02:13:16,834 ERROR wandb-AsyncioManager-main:3896698 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
|
| 23 |
+
Traceback (most recent call last):
|
| 24 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
|
| 25 |
+
await fn()
|
| 26 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
|
| 27 |
+
await asyncio_compat.race(
|
| 28 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
|
| 29 |
+
async with open_task_group(race=True) as tg:
|
| 30 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 31 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
|
| 32 |
+
await anext(self.gen)
|
| 33 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
|
| 34 |
+
await task_group._wait_all(race=race, timeout=exit_timeout)
|
| 35 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
|
| 36 |
+
raise exc
|
| 37 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
|
| 38 |
+
result = await handle.wait_async(timeout=None)
|
| 39 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 40 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
|
| 41 |
+
response = await self._handle.wait_async(timeout=timeout)
|
| 42 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 43 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
|
| 44 |
+
raise HandleAbandonedError()
|
| 45 |
+
wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
|
Abs_Joint/Baseline/wandb/run-20260717_154830-a3ogbmg4/run-a3ogbmg4.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c3b55846504f4d60e38f9eccea1b59427cedb9328f74b91e3c5a48d4504bd7dd
|
| 3 |
+
size 36212873
|