Hugging Face's logo

USC-PSI-Lab
/
psi-model

Safetensors
English
Model card Files
xet
 Community
jie530 commited on
Commit
4cb1d65
·
verified ·
1 Parent(s): 02c5669

Delete act/act-g1-sim

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/argv.txt +0 -40
  2. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/model.safetensors +0 -3
  3. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/optimizer.bin +0 -3
  4. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_0.pkl +0 -3
  5. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/scheduler.bin +0 -3
  6. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/envs.txt +0 -17
  7. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/run_config.json +0 -310
  8. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug-internal.log +0 -0
  9. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug.log +0 -25
  10. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/config.yaml +0 -448
  11. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/output.log +0 -203
  12. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/requirements.txt +0 -199
  13. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-metadata.json +0 -132
  14. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-summary.json +0 -1
  15. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log +0 -0
  16. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log +0 -25
  17. act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/run-agq65opl.wandb +0 -3
  18. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/argv.txt +0 -40
  19. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/model.safetensors +0 -3
  20. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/optimizer.bin +0 -3
  21. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_0.pkl +0 -3
  22. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_1.pkl +0 -3
  23. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_2.pkl +0 -3
  24. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/scheduler.bin +0 -3
  25. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/envs.txt +0 -17
  26. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/run_config.json +0 -310
  27. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug-internal.log +0 -0
  28. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug.log +0 -25
  29. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/config.yaml +0 -448
  30. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/output.log +0 -179
  31. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/requirements.txt +0 -219
  32. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-metadata.json +0 -132
  33. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-summary.json +0 -1
  34. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-core.log +0 -19
  35. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log +0 -0
  36. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log +0 -25
  37. act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/run-26s2ml9t.wandb +0 -3
  38. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/argv.txt +0 -40
  39. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/model.safetensors +0 -3
  40. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/optimizer.bin +0 -3
  41. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_0.pkl +0 -3
  42. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/scheduler.bin +0 -3
  43. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/envs.txt +0 -17
  44. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/run_config.json +0 -310
  45. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug-internal.log +0 -0
  46. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug.log +0 -25
  47. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/config.yaml +0 -448
  48. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/output.log +0 -237
  49. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/requirements.txt +0 -219
  50. act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/wandb-metadata.json +0 -132
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/argv.txt DELETED
@@ -1,40 +0,0 @@
1
- scripts/train.py
2
- simple_act_config
3
- --seed=2026
4
- --exp=g1wholebodybendpick-v0
5
- --train.name=act-g1
6
- --log.report-to=wandb
7
- --train.data_parallel=ddp
8
- --train.mixed_precision=bf16
9
- --train.train-batch-size=32
10
- --train.warmup-steps=1000
11
- --train.warmup-ratio=None
12
- --train.checkpointing-steps=5000
13
- --train.validation_steps=500
14
- --train.val_num_batches=20
15
- --train.gradient_accumulation_steps=1
16
- --train.max-training-steps=40000
17
- --train.learning-rate=1e-4
18
- --train.max-grad-norm=1.0
19
- --train.lr_scheduler_kwargs.weight_decay=1e-6
20
- --train.lr_scheduler_kwargs.betas 0.95 0.999
21
- --train.lr_scheduler_type=cosine
22
- --data.root_dir=/data/jliu/data
23
- --data.train-repo-ids=G1WholebodyBendPick-v0
24
- --data.transform.repack.action_chunk_size=30
25
- --data.transform.repack.pad-action-dim=36
26
- --data.transform.repack.pad-state-dim=36
27
- --data.transform.field.stat-path=meta/stats_psi0.json
28
- --data.transform.field.stat-action-key=action
29
- --data.transform.field.stat-state-key=states
30
- --data.transform.field.normalize-state
31
- --data.transform.field.action-norm-type=bounds
32
- --data.transform.field.pad-action-dim=36
33
- --data.transform.field.pad-state-dim=36
34
- --data.transform.model.img-aug
35
- --model.chunk-size=30
36
- --model.n-action-steps=30
37
- --model.action-dim=36
38
- --model.state-dim=36
39
- --model.use-vae
40
- --model.kl-weight=10.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:fb015062b468f83f0f8d65b9e097353c9e1501e127f17ec8a509150866d178e7
3
- size 206658952
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/optimizer.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e333a033a30c0ed624973e1e81ae16152f89080f9d33944ed80782e04945056
3
- size 413111371
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_states_0.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8ce2a0461b67075096482bb9b83ea22fa0f07f2d3b9e6640249cdb0e3ddd117d
3
- size 15473
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/scheduler.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:edd2ef4de5774723b53b76baced1d134ab2a534e3951c7a3ebfbc157468c8f72
3
- size 1401
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/envs.txt DELETED
@@ -1,17 +0,0 @@
1
- OMP_NUM_THREADS=32
2
- HF_HOME=/data/cache
3
- TORCH_HOME=/data/cache
4
- HF_TOKEN=hf_...TiKa
5
- HF_LEROBOT_HOME=/data/data/lerobot
6
- WE_HOME=Not Set
7
- DATA_HOME=/data/data
8
- UV_CACHE_DIR=/data/cache
9
- WANDB_API_KEY=90e...5c06
10
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
11
- CUDA_VISIBLE_DEVICES=0,1,2,3
12
- WORLD_SIZE=4
13
- LOCAL_WORLD_SIZE=4
14
- RANK=0
15
- LOCAL_RANK=0
16
- MASTER_ADDR=nebula99
17
- MASTER_PORT=37919
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/run_config.json DELETED
@@ -1,310 +0,0 @@
1
- {
2
- "exp": "g1wholebodybendpick-v0",
3
- "seed": 2026,
4
- "auto_tag_run": false,
5
- "eval": false,
6
- "debug": false,
7
- "timestamp": "2603181426",
8
- "log": {
9
- "logging_dir": "logs",
10
- "report_to": "wandb",
11
- "log_freq": 100
12
- },
13
- "wandb": {
14
- "project": "psi",
15
- "entity": "jliu530-soochow-university",
16
- "group": "act-g1",
17
- "id": "agq65opl",
18
- "name": "g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426",
19
- "resume": "allow"
20
- },
21
- "train": {
22
- "num_workers": 8,
23
- "overfit_single_batch": false,
24
- "name": "act-g1",
25
- "resume_from_checkpoint": null,
26
- "skip_resumed_steps": false,
27
- "hf_token": ".hf_token",
28
- "lora": false,
29
- "output_dir": ".runs",
30
- "gradient_accumulation_steps": 1,
31
- "mixed_precision": "bf16",
32
- "max_grad_norm": 1.0,
33
- "optimizer_foreach": null,
34
- "train_batch_size": 32,
35
- "val_batch_size": 16,
36
- "val_num_batches": 20,
37
- "checkpointing_steps": 5000,
38
- "max_checkpoints_to_keep": null,
39
- "validation_steps": 500,
40
- "learning_rate": 0.0001,
41
- "lr_scheduler_type": "cosine",
42
- "lr_scheduler_kwargs": {
43
- "betas": [
44
- 0.95,
45
- 0.999
46
- ],
47
- "weight_decay": 1e-6,
48
- "eps": 1e-8
49
- },
50
- "scheduler_specific_kwargs": {},
51
- "data_parallel": "ddp",
52
- "sharding_strategy": "full-shard",
53
- "deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
54
- "enable_gradient_checkpointing": true,
55
- "enable_mixed_precision_training": true,
56
- "reduce_in_full_precision": true,
57
- "max_training_steps": 40000,
58
- "num_train_epochs": null,
59
- "warmup_steps": 1000,
60
- "warmup_ratio": null
61
- },
62
- "data": {
63
- "transform": {
64
- "repack": {
65
- "dataset_name": "simple",
66
- "num_past_frames": 0,
67
- "action_chunk_size": 30,
68
- "pad_action_dim": 36,
69
- "pad_state_dim": 36
70
- },
71
- "model": {
72
- "resize": {
73
- "size": [
74
- 256,
75
- 480
76
- ]
77
- },
78
- "center_crop": {
79
- "size": [
80
- 224,
81
- 224
82
- ]
83
- },
84
- "color_jitter": {
85
- "brightness": 0.2,
86
- "contrast": [
87
- 0.8,
88
- 1.2
89
- ],
90
- "saturation": [
91
- 0.8,
92
- 1.2
93
- ],
94
- "hue": 0.05
95
- },
96
- "normalize": {
97
- "mean": [
98
- 0.485,
99
- 0.456,
100
- 0.406
101
- ],
102
- "std": [
103
- 0.229,
104
- 0.224,
105
- 0.225
106
- ]
107
- },
108
- "img_aug": true
109
- },
110
- "field": {
111
- "stat_path": "meta/stats_psi0.json",
112
- "action_norm_type": "bounds",
113
- "stat_action_key": "action",
114
- "stat_state_key": "states",
115
- "use_norm_mask": false,
116
- "action_norm_masks": [
117
- true,
118
- true,
119
- true,
120
- true,
121
- true,
122
- true,
123
- false
124
- ],
125
- "action_min": [
126
- -0.13059291243553162,
127
- -0.09108058363199234,
128
- -0.0024844733998179436,
129
- -0.20733775198459625,
130
- -0.15850023925304413,
131
- -0.17450474202632904,
132
- -0.2997315526008606,
133
- -0.015391111373901367,
134
- -0.34571564197540283,
135
- -0.4991437792778015,
136
- 0.0,
137
- 0.0,
138
- 0.0,
139
- 0.0,
140
- -0.1015840545296669,
141
- -0.06647031009197235,
142
- -0.16578954458236694,
143
- -0.14477218687534332,
144
- -0.3665394186973572,
145
- -0.28364259004592896,
146
- -0.1775387078523636,
147
- -0.48419490456581116,
148
- -0.7551082968711853,
149
- -0.2692946195602417,
150
- -0.03164339065551758,
151
- -0.00003876500704791397,
152
- -0.3909206688404083,
153
- 0.0,
154
- -0.04351663216948509,
155
- -0.014203650876879692,
156
- -0.049649015069007874,
157
- 0.44999998807907104,
158
- 0.0,
159
- 0.0,
160
- 0.0,
161
- 0.0
162
- ],
163
- "action_max": [
164
- 0.08620641380548477,
165
- 0.13058121502399445,
166
- 0.22948147356510162,
167
- 0.020551620051264763,
168
- 0.005824880674481392,
169
- 0.010019193403422832,
170
- 8.43817247186962e-7,
171
- 0.39566752314567566,
172
- 0.0,
173
- 0.0,
174
- 0.4860266447067261,
175
- 1.0467392206192017,
176
- 0.6470075845718384,
177
- 0.8298009037971497,
178
- 0.03516175225377083,
179
- 0.11019192636013031,
180
- 0.04779902100563049,
181
- 0.12850724160671234,
182
- 0.000038688118365826085,
183
- 0.0012142359046265483,
184
- 0.000033343669201713055,
185
- 0.002679983852431178,
186
- 0.00041063950629904866,
187
- 0.1973484456539154,
188
- 0.2633756697177887,
189
- 0.34943076968193054,
190
- 0.0012102096807211637,
191
- 0.8342975974082947,
192
- 0.31870752573013306,
193
- 0.45533719658851624,
194
- 0.15729404985904694,
195
- 0.75,
196
- 0.0,
197
- 0.0,
198
- 0.0,
199
- 0.0
200
- ],
201
- "state_min": [
202
- -0.13899999856948853,
203
- -0.09099991619586945,
204
- -5.989517215532203e-11,
205
- -0.20900000631809235,
206
- -0.1589999943971634,
207
- -0.210999995470047,
208
- -0.3009999990463257,
209
- -0.01600000075995922,
210
- -0.1860000044107437,
211
- -0.6940000057220459,
212
- 0.0,
213
- 0.0,
214
- 0.0,
215
- 0.0,
216
- -0.1019991859793663,
217
- -0.06899992376565933,
218
- -0.16899999976158142,
219
- -0.14499999582767487,
220
- -0.3709999918937683,
221
- -0.28700000047683716,
222
- -0.17800045013427734,
223
- -0.4869999885559082,
224
- -0.7599999904632568,
225
- -0.27300000190734863,
226
- -0.029999999329447746,
227
- 0.0,
228
- -0.39100033044815063,
229
- -0.0010000000474974513,
230
- 0.0,
231
- -0.15000000596046448,
232
- 0.0,
233
- 0.44999998807907104,
234
- 0.0,
235
- 0.0,
236
- 0.0,
237
- 0.0
238
- ],
239
- "state_max": [
240
- 0.0860000029206276,
241
- 0.2720000147819519,
242
- 0.23100000619888306,
243
- 0.0,
244
- 6.510182259944486e-8,
245
- 0.0,
246
- 0.0,
247
- 0.5550000071525574,
248
- 0.02100004442036152,
249
- 0.0,
250
- 0.5429999828338623,
251
- 1.13100004196167,
252
- 0.5770000219345093,
253
- 0.9580000042915344,
254
- 0.07141251862049103,
255
- 0.10899999737739563,
256
- 0.04699999839067459,
257
- 0.13600000739097595,
258
- 0.0,
259
- 0.003000000026077032,
260
- 0.0,
261
- 0.009999999776482582,
262
- 0.0020000000949949026,
263
- 0.2029999941587448,
264
- 0.2759999930858612,
265
- 0.3499999940395355,
266
- 0.003000000026077032,
267
- 0.8370000123977661,
268
- 0.0,
269
- 0.0,
270
- 0.0,
271
- 0.75,
272
- 0.0,
273
- 0.0,
274
- 0.0,
275
- 0.0
276
- ],
277
- "normalize_state": true,
278
- "pad_action_dim": 36,
279
- "pad_state_dim": 36
280
- }
281
- },
282
- "root_dir": "/data/jliu/data",
283
- "train_repo_ids": [
284
- "G1WholebodyBendPick-v0"
285
- ],
286
- "val_repo_ids": [
287
- "G1WholebodyBendPick-v0"
288
- ]
289
- },
290
- "model": {
291
- "n_obs_steps": 1,
292
- "chunk_size": 30,
293
- "n_action_steps": 30,
294
- "action_dim": 36,
295
- "state_dim": 36,
296
- "dim_model": 512,
297
- "n_heads": 8,
298
- "dim_feedforward": 3200,
299
- "feedforward_activation": "relu",
300
- "n_encoder_layers": 4,
301
- "n_decoder_layers": 1,
302
- "pre_norm": false,
303
- "dropout": 0.1,
304
- "use_vae": true,
305
- "latent_dim": 32,
306
- "n_vae_encoder_layers": 4,
307
- "kl_weight": 10.0,
308
- "temporal_ensemble_coeff": null
309
- }
310
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug-internal.log DELETED
The diff for this file is too large to render. See raw diff
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/debug.log DELETED
@@ -1,25 +0,0 @@
1
- 2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
2
- 2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Configure stats pid to 280442
3
- 2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
- 2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log
5
- 2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log
6
- 2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():844] calling init triggers
7
- 2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
- config: {'_wandb': {}}
9
- 2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():892] starting backend
10
- 2026-03-18 14:26:46,802 INFO MainThread:280442 [wandb_init.py:init():895] sending inform_init request
11
- 2026-03-18 14:26:46,814 INFO MainThread:280442 [wandb_init.py:init():903] backend started and connected
12
- 2026-03-18 14:26:46,815 INFO MainThread:280442 [wandb_init.py:init():973] updated telemetry
13
- 2026-03-18 14:26:46,822 INFO MainThread:280442 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
- 2026-03-18 14:26:48,282 INFO MainThread:280442 [wandb_init.py:init():1042] starting run threads in backend
15
- 2026-03-18 14:26:48,444 INFO MainThread:280442 [wandb_run.py:_console_start():2524] atexit reg
16
- 2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
- 2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
- 2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2465] Redirects installed.
19
- 2026-03-18 14:26:48,451 INFO MainThread:280442 [wandb_init.py:init():1082] run started, returning control to user process
20
- 2026-03-18 14:26:48,454 INFO MainThread:280442 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodybendpick-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2603181426', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.13059291243553162, -0.09108058363199234, -0.0024844733998179436, -0.20733775198459625, -0.15850023925304413, -0.17450474202632904, -0.2997315526008606, -0.015391111373901367, -0.34571564197540283, -0.4991437792778015, 0.0, 0.0, 0.0, 0.0, -0.1015840545296669, -0.06647031009197235, -0.16578954458236694, -0.14477218687534332, -0.3665394186973572, -0.28364259004592896, -0.1775387078523636, -0.48419490456581116, -0.7551082968711853, -0.2692946195602417, -0.03164339065551758, -3.876500704791397e-05, -0.3909206688404083, 0.0, -0.04351663216948509, -0.014203650876879692, -0.049649015069007874, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.08620641380548477, 0.13058121502399445, 0.22948147356510162, 0.020551620051264763, 0.005824880674481392, 0.010019193403422832, 8.43817247186962e-07, 0.39566752314567566, 0.0, 0.0, 0.4860266447067261, 1.0467392206192017, 0.6470075845718384, 0.8298009037971497, 0.03516175225377083, 0.11019192636013031, 0.04779902100563049, 0.12850724160671234, 3.8688118365826085e-05, 0.0012142359046265483, 3.3343669201713055e-05, 0.002679983852431178, 0.00041063950629904866, 0.1973484456539154, 0.2633756697177887, 0.34943076968193054, 0.0012102096807211637, 0.8342975974082947, 0.31870752573013306, 0.45533719658851624, 0.15729404985904694, 0.75, 0.0, 0.0, 0.0, 0.0], 'state_min': [-0.13899999856948853, -0.09099991619586945, -5.989517215532203e-11, -0.20900000631809235, -0.1589999943971634, -0.210999995470047, -0.3009999990463257, -0.01600000075995922, -0.1860000044107437, -0.6940000057220459, 0.0, 0.0, 0.0, 0.0, -0.1019991859793663, -0.06899992376565933, -0.16899999976158142, -0.14499999582767487, -0.3709999918937683, -0.28700000047683716, -0.17800045013427734, -0.4869999885559082, -0.7599999904632568, -0.27300000190734863, -0.029999999329447746, 0.0, -0.39100033044815063, -0.0010000000474974513, 0.0, -0.15000000596046448, 0.0, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.0860000029206276, 0.2720000147819519, 0.23100000619888306, 0.0, 6.510182259944486e-08, 0.0, 0.0, 0.5550000071525574, 0.02100004442036152, 0.0, 0.5429999828338623, 1.13100004196167, 0.5770000219345093, 0.9580000042915344, 0.07141251862049103, 0.10899999737739563, 0.04699999839067459, 0.13600000739097595, 0.0, 0.003000000026077032, 0.0, 0.009999999776482582, 0.0020000000949949026, 0.2029999941587448, 0.2759999930858612, 0.3499999940395355, 0.003000000026077032, 0.8370000123977661, 0.0, 0.0, 0.0, 0.75, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyBendPick-v0'], 'val_repo_ids': ['G1WholebodyBendPick-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '0,1,2,3', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '37919'}}
21
- 2026-03-18 19:53:48,477 INFO MainThread:280442 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/agq65opl
22
- 2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
23
- 2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2472] restore
24
- 2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2478] restore done
25
- 2026-03-18 19:53:50,040 INFO MainThread:280442 [wandb_run.py:_footer_sync_info():3868] logging synced files
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/config.yaml DELETED
@@ -1,448 +0,0 @@
1
- _wandb:
2
- value:
3
- cli_version: 0.25.1
4
- e:
5
- 0fepodeqeh00upnrzhqar4dfcf23o49l:
6
- args:
7
- - simple_act_config
8
- - --seed=2026
9
- - --exp=g1wholebodybendpick-v0
10
- - --train.name=act-g1
11
- - --log.report-to=wandb
12
- - --train.data_parallel=ddp
13
- - --train.mixed_precision=bf16
14
- - --train.train-batch-size=32
15
- - --train.warmup-steps=1000
16
- - --train.warmup-ratio=None
17
- - --train.checkpointing-steps=5000
18
- - --train.validation_steps=500
19
- - --train.val_num_batches=20
20
- - --train.gradient_accumulation_steps=1
21
- - --train.max-training-steps=40000
22
- - --train.learning-rate=1e-4
23
- - --train.max-grad-norm=1.0
24
- - --train.lr_scheduler_kwargs.weight_decay=1e-6
25
- - --train.lr_scheduler_kwargs.betas
26
- - "0.95"
27
- - "0.999"
28
- - --train.lr_scheduler_type=cosine
29
- - --data.root_dir=/data/jliu/data
30
- - --data.train-repo-ids=G1WholebodyBendPick-v0
31
- - --data.transform.repack.action_chunk_size=30
32
- - --data.transform.repack.pad-action-dim=36
33
- - --data.transform.repack.pad-state-dim=36
34
- - --data.transform.field.stat-path=meta/stats_psi0.json
35
- - --data.transform.field.stat-action-key=action
36
- - --data.transform.field.stat-state-key=states
37
- - --data.transform.field.normalize-state
38
- - --data.transform.field.action-norm-type=bounds
39
- - --data.transform.field.pad-action-dim=36
40
- - --data.transform.field.pad-state-dim=36
41
- - --data.transform.model.img-aug
42
- - --model.chunk-size=30
43
- - --model.n-action-steps=30
44
- - --model.action-dim=36
45
- - --model.state-dim=36
46
- - --model.use-vae
47
- - --model.kl-weight=10.0
48
- codePath: scripts/train.py
49
- codePathLocal: scripts/train.py
50
- cpu_count: 128
51
- cpu_count_logical: 128
52
- cudaVersion: "12.9"
53
- disk:
54
- /:
55
- total: "105089261568"
56
- used: "99538649088"
57
- email: jliu530@163.com
58
- executable: /data/jliu/psi/.venv-psi/bin/python3
59
- git:
60
- commit: 04614628ecb677f5f278e2d31c1103aed8127f26
61
- remote: https://github.com/songlin/psi.git
62
- gpu: NVIDIA A100-SXM4-80GB
63
- gpu_count: 8
64
- gpu_nvidia:
65
- - architecture: Ampere
66
- cudaCores: 6912
67
- memoryTotal: "85899345920"
68
- name: NVIDIA A100-SXM4-80GB
69
- uuid: GPU-37370f00-90f5-4e03-f763-7a35649e6783
70
- - architecture: Ampere
71
- cudaCores: 6912
72
- memoryTotal: "85899345920"
73
- name: NVIDIA A100-SXM4-80GB
74
- uuid: GPU-d4b40383-188b-cc16-9180-20c1a71a777f
75
- - architecture: Ampere
76
- cudaCores: 6912
77
- memoryTotal: "85899345920"
78
- name: NVIDIA A100-SXM4-80GB
79
- uuid: GPU-0f3d827c-66cf-04da-b182-4ed0414a2549
80
- - architecture: Ampere
81
- cudaCores: 6912
82
- memoryTotal: "85899345920"
83
- name: NVIDIA A100-SXM4-80GB
84
- uuid: GPU-164a49bb-43d3-2250-59e2-1b40eee0757f
85
- - architecture: Ampere
86
- cudaCores: 6912
87
- memoryTotal: "85899345920"
88
- name: NVIDIA A100-SXM4-80GB
89
- uuid: GPU-c0471f40-ae19-5371-a1af-da406ca30f83
90
- - architecture: Ampere
91
- cudaCores: 6912
92
- memoryTotal: "85899345920"
93
- name: NVIDIA A100-SXM4-80GB
94
- uuid: GPU-d9040f59-13d3-6c37-4ea0-c6c751424527
95
- - architecture: Ampere
96
- cudaCores: 6912
97
- memoryTotal: "85899345920"
98
- name: NVIDIA A100-SXM4-80GB
99
- uuid: GPU-f834d0e8-a118-24c7-5461-30fce111bea6
100
- - architecture: Ampere
101
- cudaCores: 6912
102
- memoryTotal: "85899345920"
103
- name: NVIDIA A100-SXM4-80GB
104
- uuid: GPU-6e687915-3247-735c-6641-544540804e79
105
- host: nebula99
106
- memory:
107
- total: "1623177744384"
108
- os: Linux-6.8.0-100-generic-x86_64-with-glibc2.39
109
- program: /data/jliu/psi/scripts/train.py
110
- python: CPython 3.10.20
111
- root: /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426
112
- startedAt: "2026-03-18T14:26:46.537089Z"
113
- writerId: 0fepodeqeh00upnrzhqar4dfcf23o49l
114
- m: []
115
- python_version: 3.10.20
116
- t:
117
- "1":
118
- - 1
119
- - 11
120
- - 41
121
- - 49
122
- - 71
123
- "2":
124
- - 1
125
- - 11
126
- - 41
127
- - 49
128
- - 51
129
- - 71
130
- - 83
131
- "3":
132
- - 2
133
- - 13
134
- - 61
135
- "4": 3.10.20
136
- "5": 0.25.1
137
- "6": 4.57.0
138
- "12": 0.25.1
139
- "13": linux-x86_64
140
- auto_tag_run:
141
- value: false
142
- data:
143
- value:
144
- root_dir: /data/jliu/data
145
- train_repo_ids:
146
- - G1WholebodyBendPick-v0
147
- transform:
148
- field:
149
- action_max:
150
- - 0.08620641380548477
151
- - 0.13058121502399445
152
- - 0.22948147356510162
153
- - 0.020551620051264763
154
- - 0.005824880674481392
155
- - 0.010019193403422832
156
- - 8.43817247186962e-07
157
- - 0.39566752314567566
158
- - 0
159
- - 0
160
- - 0.4860266447067261
161
- - 1.0467392206192017
162
- - 0.6470075845718384
163
- - 0.8298009037971497
164
- - 0.03516175225377083
165
- - 0.11019192636013031
166
- - 0.04779902100563049
167
- - 0.12850724160671234
168
- - 3.8688118365826085e-05
169
- - 0.0012142359046265483
170
- - 3.3343669201713055e-05
171
- - 0.002679983852431178
172
- - 0.00041063950629904866
173
- - 0.1973484456539154
174
- - 0.2633756697177887
175
- - 0.34943076968193054
176
- - 0.0012102096807211637
177
- - 0.8342975974082947
178
- - 0.31870752573013306
179
- - 0.45533719658851624
180
- - 0.15729404985904694
181
- - 0.75
182
- - 0
183
- - 0
184
- - 0
185
- - 0
186
- action_min:
187
- - -0.13059291243553162
188
- - -0.09108058363199234
189
- - -0.0024844733998179436
190
- - -0.20733775198459625
191
- - -0.15850023925304413
192
- - -0.17450474202632904
193
- - -0.2997315526008606
194
- - -0.015391111373901367
195
- - -0.34571564197540283
196
- - -0.4991437792778015
197
- - 0
198
- - 0
199
- - 0
200
- - 0
201
- - -0.1015840545296669
202
- - -0.06647031009197235
203
- - -0.16578954458236694
204
- - -0.14477218687534332
205
- - -0.3665394186973572
206
- - -0.28364259004592896
207
- - -0.1775387078523636
208
- - -0.48419490456581116
209
- - -0.7551082968711853
210
- - -0.2692946195602417
211
- - -0.03164339065551758
212
- - -3.876500704791397e-05
213
- - -0.3909206688404083
214
- - 0
215
- - -0.04351663216948509
216
- - -0.014203650876879692
217
- - -0.049649015069007874
218
- - 0.44999998807907104
219
- - 0
220
- - 0
221
- - 0
222
- - 0
223
- action_norm_masks:
224
- - true
225
- - true
226
- - true
227
- - true
228
- - true
229
- - true
230
- - false
231
- action_norm_type: bounds
232
- normalize_state: true
233
- pad_action_dim: 36
234
- pad_state_dim: 36
235
- stat_action_key: action
236
- stat_path: meta/stats_psi0.json
237
- stat_state_key: states
238
- state_max:
239
- - 0.0860000029206276
240
- - 0.2720000147819519
241
- - 0.23100000619888306
242
- - 0
243
- - 6.510182259944486e-08
244
- - 0
245
- - 0
246
- - 0.5550000071525574
247
- - 0.02100004442036152
248
- - 0
249
- - 0.5429999828338623
250
- - 1.13100004196167
251
- - 0.5770000219345093
252
- - 0.9580000042915344
253
- - 0.07141251862049103
254
- - 0.10899999737739563
255
- - 0.04699999839067459
256
- - 0.13600000739097595
257
- - 0
258
- - 0.003000000026077032
259
- - 0
260
- - 0.009999999776482582
261
- - 0.0020000000949949026
262
- - 0.2029999941587448
263
- - 0.2759999930858612
264
- - 0.3499999940395355
265
- - 0.003000000026077032
266
- - 0.8370000123977661
267
- - 0
268
- - 0
269
- - 0
270
- - 0.75
271
- - 0
272
- - 0
273
- - 0
274
- - 0
275
- state_min:
276
- - -0.13899999856948853
277
- - -0.09099991619586945
278
- - -5.989517215532203e-11
279
- - -0.20900000631809235
280
- - -0.1589999943971634
281
- - -0.210999995470047
282
- - -0.3009999990463257
283
- - -0.01600000075995922
284
- - -0.1860000044107437
285
- - -0.6940000057220459
286
- - 0
287
- - 0
288
- - 0
289
- - 0
290
- - -0.1019991859793663
291
- - -0.06899992376565933
292
- - -0.16899999976158142
293
- - -0.14499999582767487
294
- - -0.3709999918937683
295
- - -0.28700000047683716
296
- - -0.17800045013427734
297
- - -0.4869999885559082
298
- - -0.7599999904632568
299
- - -0.27300000190734863
300
- - -0.029999999329447746
301
- - 0
302
- - -0.39100033044815063
303
- - -0.0010000000474974513
304
- - 0
305
- - -0.15000000596046448
306
- - 0
307
- - 0.44999998807907104
308
- - 0
309
- - 0
310
- - 0
311
- - 0
312
- use_norm_mask: false
313
- model:
314
- center_crop:
315
- size:
316
- - 224
317
- - 224
318
- color_jitter:
319
- brightness: 0.2
320
- contrast:
321
- - 0.8
322
- - 1.2
323
- hue: 0.05
324
- saturation:
325
- - 0.8
326
- - 1.2
327
- img_aug: true
328
- normalize:
329
- mean:
330
- - 0.485
331
- - 0.456
332
- - 0.406
333
- std:
334
- - 0.229
335
- - 0.224
336
- - 0.225
337
- resize:
338
- size:
339
- - 256
340
- - 480
341
- repack:
342
- action_chunk_size: 30
343
- dataset_name: simple
344
- num_past_frames: 0
345
- pad_action_dim: 36
346
- pad_state_dim: 36
347
- val_repo_ids:
348
- - G1WholebodyBendPick-v0
349
- debug:
350
- value: false
351
- environment_variables:
352
- value:
353
- CUDA_VISIBLE_DEVICES: 0,1,2,3
354
- DATA_HOME: /data/data
355
- HF_HOME: /data/cache
356
- HF_LEROBOT_HOME: /data/data/lerobot
357
- HF_TOKEN: hf_...TiKa
358
- LOCAL_RANK: "0"
359
- LOCAL_WORLD_SIZE: "4"
360
- MASTER_ADDR: nebula99
361
- MASTER_PORT: "37919"
362
- OMP_NUM_THREADS: "32"
363
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
364
- RANK: "0"
365
- TORCH_HOME: /data/cache
366
- UV_CACHE_DIR: /data/cache
367
- WANDB_API_KEY: 90e...5c06
368
- WE_HOME: Not Set
369
- WORLD_SIZE: "4"
370
- eval:
371
- value: false
372
- exp:
373
- value: g1wholebodybendpick-v0
374
- log:
375
- value:
376
- log_freq: 100
377
- logging_dir: logs
378
- report_to: wandb
379
- model:
380
- value:
381
- action_dim: 36
382
- chunk_size: 30
383
- dim_feedforward: 3200
384
- dim_model: 512
385
- dropout: 0.1
386
- feedforward_activation: relu
387
- kl_weight: 10
388
- latent_dim: 32
389
- n_action_steps: 30
390
- n_decoder_layers: 1
391
- n_encoder_layers: 4
392
- n_heads: 8
393
- n_obs_steps: 1
394
- n_vae_encoder_layers: 4
395
- pre_norm: false
396
- state_dim: 36
397
- temporal_ensemble_coeff: null
398
- use_vae: true
399
- seed:
400
- value: 2026
401
- timestamp:
402
- value: "2603181426"
403
- train:
404
- value:
405
- checkpointing_steps: 5000
406
- data_parallel: ddp
407
- deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
408
- enable_gradient_checkpointing: true
409
- enable_mixed_precision_training: true
410
- gradient_accumulation_steps: 1
411
- hf_token: .hf_token
412
- learning_rate: 0.0001
413
- lora: false
414
- lr_scheduler_kwargs:
415
- betas:
416
- - 0.95
417
- - 0.999
418
- eps: 1e-08
419
- weight_decay: 1e-06
420
- lr_scheduler_type: cosine
421
- max_checkpoints_to_keep: null
422
- max_grad_norm: 1
423
- max_training_steps: 40000
424
- mixed_precision: bf16
425
- name: act-g1
426
- num_train_epochs: null
427
- num_workers: 8
428
- optimizer_foreach: null
429
- output_dir: .runs
430
- overfit_single_batch: false
431
- reduce_in_full_precision: true
432
- resume_from_checkpoint: null
433
- sharding_strategy: full-shard
434
- skip_resumed_steps: false
435
- train_batch_size: 32
436
- val_batch_size: 16
437
- val_num_batches: 20
438
- validation_steps: 500
439
- warmup_ratio: null
440
- warmup_steps: 1000
441
- wandb:
442
- value:
443
- entity: jliu530-soochow-university
444
- group: null
445
- id: null
446
- name: null
447
- project: psi
448
- resume: allow
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/output.log DELETED
@@ -1,203 +0,0 @@
1
- [14:26:48 03/18] INFO  | >> [*] Saved configuration to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426 ]8;id=805427;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\128]8;;\
2
-   INFO  | >> [*] Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\181]8;;\
3
-   INFO  | >> |=> training task: 'act-g1' ]8;id=471497;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\182]8;;\
4
-   INFO  | >> |=> run name: g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426 ]8;id=329497;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\183]8;;\
5
-   INFO  | >> |=> seed: 2026 ]8;id=844246;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\184]8;;\
6
-   INFO  | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\185]8;;\
7
-   INFO  | >> |=> warmup steps: 1000 ]8;id=147732;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\186]8;;\
8
-   INFO  | >> |=> validation steps: 500 ]8;id=25941;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\187]8;;\
9
-   INFO  | >> |=> checkpoint steps: 5000 ]8;id=95089;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\188]8;;\
10
-   INFO  | >> |=> max gradient norm: 1.0 ]8;id=122104;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\189]8;;\
11
-   INFO  | >> PyTorch version 2.7.0 available. ]8;id=514883;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py\config.py]8;;\:]8;id=417488;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py#54\54]8;;\
12
- Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 206108.30it/s]
13
- Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 191084.46it/s]
14
- [14:26:50 03/18] INFO  | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\192]8;;\
15
-   INFO  | >> |=> Training dataset size: 15,509 ]8;id=603472;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\193]8;;\
16
-   INFO  | >> |=> Val dataset size: 15,509 ]8;id=548632;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\195]8;;\
17
-   INFO  | >> [*] Initialize optimizers and schedulers... ]8;id=492873;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\198]8;;\
18
-   INFO  | >> [*] ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\202]8;;\
19
-   INFO  | >> |=> Num training examples = 15509 ]8;id=974614;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\203]8;;\
20
-   INFO  | >> |=> Max training Epochs = 328 ]8;id=574775;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\204]8;;\
21
-   INFO  | >> |=> Total optimization steps = 40000 ]8;id=927413;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\205]8;;\
22
-   INFO  | >> |=> Num steps Per Epoch = 122 ]8;id=9727;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\206]8;;\
23
-   INFO  | >> |=> Effective training epochs = 1311.48 ]8;id=842580;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\207]8;;\
24
-   INFO  | >> |=> Global train batch size (w. parallel, distributed & accumulation) = 128 ]8;id=272745;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\208]8;;\
25
-   INFO  | >> |=> Device train batch size = 32 ]8;id=521307;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\209]8;;\
26
-   INFO  | >> |=> Gradient Accumulation steps = 1 ]8;id=871789;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\210]8;;\
27
-   INFO  | >> |=> Num processes (GPUs) = 4 ]8;id=459749;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\211]8;;\
28
- [14:26:51 03/18] INFO  | >> [*] Accelerator runs in: .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426 ]8;id=966756;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\218]8;;\
29
- Traing steps: 25%|██████████████████████▍ | 9999/40000 [1:25:29<2:57:03, 2.82it/s, loss=0.0482, lr=8.7e-05]
30
- [2026-03-18 14:26:59,782] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
31
- [14:26:59 03/18] INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=680052;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=279409;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
32
-   /tmp/tmpb55feb63/test.c -o /tmp/tmpb55feb63/test.o  
33
-   INFO  | >> cc -pthread /tmp/tmpb55feb63/test.o -laio -o /tmp/tmpb55feb63/a.out ]8;id=204038;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=783864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
34
- [14:27:00 03/18] INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=960999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=14714;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
35
-   /tmp/tmp54jhxf13/test.c -o /tmp/tmp54jhxf13/test.o  
36
-   INFO  | >> cc -pthread /tmp/tmp54jhxf13/test.o -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o /tmp/tmp54jhxf13/a.out ]8;id=573141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=977459;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
37
-   INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=208448;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=259876;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
38
-   /tmp/tmpksfo7p26/test.c -o /tmp/tmpksfo7p26/test.o  
39
-   INFO  | >> cc -pthread /tmp/tmpksfo7p26/test.o -laio -o /tmp/tmpksfo7p26/a.out ]8;id=572992;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=281580;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
40
- [2026-03-18 14:27:01,308] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
41
-
42
- [15:09:33 03/18] INFO  | >> Saving current state to ]8;id=328668;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=696864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
43
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000  
44
- [15:09:34 03/18] INFO  | >> Model weights saved in ]8;id=515887;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=723262;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
45
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000/model.safeten  
46
-   sors  
47
- [15:09:35 03/18] INFO  | >> Optimizer state saved in ]8;id=495959;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=616494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
48
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000/optimizer.bin  
49
-   INFO  | >> Scheduler state saved in ]8;id=304800;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=88534;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
50
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000/scheduler.bin  
51
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=821494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=820417;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
52
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000/sampler.bin  
53
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=432054;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=409371;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
54
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000/sampler_1.bin  
55
-   INFO  | >> Random states saved in ]8;id=394271;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=14818;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
56
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000/random_states  
57
-   _0.pkl  
58
- Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_5000
59
- [15:52:21 03/18] INFO  | >> Saving current state to ]8;id=53870;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=874302;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
60
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000  
61
- [15:52:24 03/18] INFO  | >> Model weights saved in ]8;id=806914;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=990978;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
62
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000/model.safete  
63
-   nsors  
64
- [15:52:25 03/18] INFO  | >> Optimizer state saved in ]8;id=343681;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=715961;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
65
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000/optimizer.bi  
66
-   n  
67
-   INFO  | >> Scheduler state saved in ]8;id=127268;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=521739;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
68
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000/scheduler.bi  
69
-   n  
70
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=844039;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=179027;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
71
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000/sampler.bin  
72
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=454670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=653622;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
73
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000/sampler_1.bi  
74
-   n  
75
-   INFO  | >> Random states saved in ]8;id=711457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=745414;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
76
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000/random_state  
77
-   s_0.pkl  
78
- Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_10000
79
- [16:33:16 03/18] INFO  | >> Saving current state to ]8;id=204990;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=678670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
80
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000  
81
- [16:33:17 03/18] INFO  | >> Model weights saved in ]8;id=571229;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=976178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
82
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000/model.safete  
83
-   nsors  
84
-   INFO  | >> Optimizer state saved in ]8;id=791370;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=671982;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
85
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000/optimizer.bi  
86
-   n  
87
-   INFO  | >> Scheduler state saved in ]8;id=920457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=21464;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
88
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000/scheduler.bi  
89
-   n  
90
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=823560;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=925542;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
91
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000/sampler.bin  
92
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=693342;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=885391;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
93
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000/sampler_1.bi  
94
-   n  
95
-   INFO  | >> Random states saved in ]8;id=558902;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=434201;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
96
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000/random_state  
97
-   s_0.pkl  
98
- Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_15000
99
- [17:14:14 03/18] INFO  | >> Saving current state to ]8;id=196292;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=360779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
100
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000  
101
- [17:14:16 03/18] INFO  | >> Model weights saved in ]8;id=818149;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=854583;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
102
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000/model.safete  
103
-   nsors  
104
- [17:14:17 03/18] INFO  | >> Optimizer state saved in ]8;id=341614;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=544562;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
105
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000/optimizer.bi  
106
-   n  
107
-   INFO  | >> Scheduler state saved in ]8;id=835298;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=550186;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
108
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000/scheduler.bi  
109
-   n  
110
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=889056;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=224742;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
111
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000/sampler.bin  
112
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=921300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=476172;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
113
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000/sampler_1.bi  
114
-   n  
115
-   INFO  | >> Random states saved in ]8;id=821364;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=874045;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
116
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000/random_state  
117
-   s_0.pkl  
118
- Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_20000
119
- [17:54:01 03/18] WARNING  | >> [*] Checkpoint 25000 already exists, skipping save. ]8;id=369046;file:///data/jliu/psi/src/psi/trainers/act_g1.py\act_g1.py]8;;\:]8;id=887661;file:///data/jliu/psi/src/psi/trainers/act_g1.py#177\177]8;;\
120
- Saved state to None
121
- [18:33:46 03/18] INFO  | >> Saving current state to ]8;id=600133;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=830265;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
122
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000  
123
- [18:33:47 03/18] INFO  | >> Model weights saved in ]8;id=750793;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=294096;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
124
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000/model.safete  
125
-   nsors  
126
-   INFO  | >> Optimizer state saved in ]8;id=224851;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=597515;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
127
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000/optimizer.bi  
128
-   n  
129
-   INFO  | >> Scheduler state saved in ]8;id=582729;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=139060;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
130
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000/scheduler.bi  
131
-   n  
132
- [18:33:48 03/18] INFO  | >> Sampler state for dataloader 0 saved in ]8;id=457234;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=179594;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
133
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000/sampler.bin  
134
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=373558;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=757447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
135
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000/sampler_1.bi  
136
-   n  
137
-   INFO  | >> Random states saved in ]8;id=111484;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=531236;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
138
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000/random_state  
139
-   s_0.pkl  
140
- Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_30000
141
- [19:13:36 03/18] INFO  | >> Saving current state to ]8;id=67019;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=305173;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
142
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000  
143
- [19:13:37 03/18] INFO  | >> Model weights saved in ]8;id=268565;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=65933;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
144
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000/model.safete  
145
-   nsors  
146
- [19:13:38 03/18] INFO  | >> Optimizer state saved in ]8;id=948866;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=814512;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
147
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000/optimizer.bi  
148
-   n  
149
-   INFO  | >> Scheduler state saved in ]8;id=101135;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=246651;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
150
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000/scheduler.bi  
151
-   n  
152
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=719141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=588438;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
153
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000/sampler.bin  
154
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=330274;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=817634;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
155
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000/sampler_1.bi  
156
-   n  
157
-   INFO  | >> Random states saved in ]8;id=748910;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=645026;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
158
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000/random_state  
159
-   s_0.pkl  
160
- Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_35000
161
- [19:53:30 03/18] INFO  | >> Saving current state to ]8;id=637309;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=226110;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
162
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000  
163
- [19:53:32 03/18] INFO  | >> Model weights saved in ]8;id=921072;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=125242;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
164
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/model.safete  
165
-   nsors  
166
- [19:53:33 03/18] INFO  | >> Optimizer state saved in ]8;id=714503;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=968603;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
167
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/optimizer.bi  
168
-   n  
169
-   INFO  | >> Scheduler state saved in ]8;id=856493;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=373626;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
170
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/scheduler.bi  
171
-   n  
172
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=921087;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=24812;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
173
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/sampler.bin  
174
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=762447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=806512;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
175
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/sampler_1.bi  
176
-   n  
177
-   INFO  | >> Random states saved in ]8;id=805194;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=436875;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
178
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000/random_state  
179
-   s_0.pkl  
180
- Saved state to .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_40000
181
- Training has reached maximum steps.
182
- [19:53:46 03/18] INFO  | >> Saving current state to ]8;id=568492;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=359120;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
183
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_39999  
184
- [19:53:47 03/18] INFO  | >> Model weights saved in ]8;id=118067;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=169999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
185
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_39999/model.safete  
186
-   nsors  
187
- [19:53:48 03/18] INFO  | >> Optimizer state saved in ]8;id=131787;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=547968;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
188
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_39999/optimizer.bi  
189
-   n  
190
-   INFO  | >> Scheduler state saved in ]8;id=316922;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=30192;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
191
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_39999/scheduler.bi  
192
-   n  
193
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=435471;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=402764;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
194
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_39999/sampler.bin  
195
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=694779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=230230;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
196
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_39999/sampler_1.bi  
197
-   n  
198
-   INFO  | >> Random states saved in ]8;id=29797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=693698;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
199
-   .runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/checkpoints/ckpt_39999/random_state  
200
-   s_0.pkl  
201
-   INFO  | >> [*] Finalized ACT Trainer. Epoch losses: [0.0, 4.91099214553833, 2.8780977725982666, 2.0492076873779297, ]8;id=416136;file:///data/jliu/psi/src/psi/trainers/act_g1.py\act_g1.py]8;;\:]8;id=932337;file:///data/jliu/psi/src/psi/trainers/act_g1.py#331\331]8;;\
202
-   1.8726189136505127, 1.5002448558807373]  
203
-   INFO  | >> [*] Happy Ending! ]8;id=17401;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=290278;file:///data/jliu/psi/scripts/train.py#310\310]8;;\
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/requirements.txt DELETED
@@ -1,199 +0,0 @@
1
- tifffile==2025.5.10
2
- mpmath==1.3.0
3
- pynput==1.8.1
4
- exceptiongroup==1.3.0
5
- sympy==1.14.0
6
- aiohttp-cors==0.8.1
7
- soupsieve==2.8
8
- Farama-Notifications==0.0.4
9
- numpydantic==1.6.7
10
- uvicorn==0.38.0
11
- waterbear==2.6.8
12
- cmake==4.2.3
13
- xxhash==3.5.0
14
- nvidia-cusparse-cu12==12.5.4.2
15
- annotated-doc==0.0.4
16
- jsonlines==4.0.0
17
- nvidia-cuda-runtime-cu12==12.6.77
18
- pydantic==2.10.6
19
- multidict==6.6.4
20
- shtab==1.7.2
21
- nvidia-cuda-nvrtc-cu12==12.6.77
22
- pycollada==0.9.2
23
- beautifulsoup4==4.14.2
24
- msgspec==0.19.0
25
- huggingface-hub==0.35.3
26
- pydantic_core==2.27.2
27
- pytz==2025.2
28
- gymnasium==1.2.3
29
- attrs==25.3.0
30
- requests==2.32.5
31
- hf_transfer==0.1.9
32
- einx==0.3.0
33
- scipy==1.15.3
34
- transforms3d==0.4.2
35
- aiohappyeyeballs==2.6.1
36
- six==1.17.0
37
- expandvars==1.1.2
38
- prompt_toolkit==3.0.52
39
- wcwidth==0.6.0
40
- ruamel.yaml.clib==0.2.14
41
- multiprocess==0.70.16
42
- vhacdx==0.0.9
43
- tokenizers==0.22.2
44
- aiosignal==1.4.0
45
- itsdangerous==2.2.0
46
- torchvision==0.22.0
47
- nvidia-cublas-cu12==12.6.4.1
48
- httpx==0.28.1
49
- platformdirs==4.4.0
50
- sentry-sdk==2.39.0
51
- plotly==6.2.0
52
- nvidia-cudnn-cu12==9.5.1.17
53
- jsonschema==4.25.1
54
- fastapi==0.119.1
55
- fsspec==2025.3.0
56
- MarkupSafe==3.0.3
57
- pydantic-yaml==1.6.0
58
- h11==0.16.0
59
- typer==0.24.1
60
- mypy_extensions==1.1.0
61
- h5py==3.14.0
62
- python-xlib==0.33
63
- lazy_loader==0.4
64
- einops==0.8.1
65
- albumentations==1.4.18
66
- params_proto==2.13.2
67
- psutil==7.1.0
68
- starlette==0.48.0
69
- anyio==4.11.0
70
- gdown==5.2.0
71
- charset-normalizer==3.4.3
72
- tyro==0.9.32
73
- filelock==3.19.1
74
- websockets==15.0.1
75
- nvidia-cuda-cupti-cu12==12.6.80
76
- python-dotenv==1.2.1
77
- orderly-set==5.5.0
78
- inquirerpy==0.3.4
79
- urllib3==2.5.0
80
- diffusers==0.37.0
81
- PyYAML==6.0.3
82
- antlr4-python3-runtime==4.9.3
83
- mdurl==0.1.2
84
- omegaconf==2.3.0
85
- rerun-sdk==0.22.1
86
- draccus==0.10.0
87
- sentencepiece==0.2.1
88
- referencing==0.37.0
89
- docstring_parser==0.17.0
90
- protobuf==6.33.5
91
- wandb==0.25.1
92
- numpy==1.26.4
93
- GitPython==3.1.45
94
- opencv-python-headless==4.11.0.86
95
- yourdfpy==0.0.58
96
- async-timeout==4.0.3
97
- shapely==2.1.2
98
- frozenlist==1.7.0
99
- simplejpeg==1.9.0
100
- Pygments==2.19.2
101
- py-cpuinfo==9.0.0
102
- rtree==1.4.1
103
- gitdb==4.0.12
104
- cloudpickle==3.1.1
105
- deepspeed==0.17.1
106
- nvidia-nvjitlink-cu12==12.6.85
107
- colorlog==6.10.1
108
- qwen-vl-utils==0.0.14
109
- Werkzeug==3.1.6
110
- zipp==3.23.0
111
- setuptools==80.9.0
112
- albucore==0.0.17
113
- transformers==4.57.0
114
- imageio==2.34.2
115
- frozendict==2.4.6
116
- hjson==3.1.0
117
- jsonschema-specifications==2025.9.1
118
- rpds-py==0.28.0
119
- blinker==1.9.0
120
- accelerate==1.7.0
121
- argparse==1.4.0
122
- msgpack==1.1.1
123
- smmap==5.0.2
124
- nvidia-nvtx-cu12==12.6.77
125
- packaging==25.0
126
- embreex==2.17.7.post7
127
- vuer==0.0.68
128
- propcache==0.3.2
129
- nvidia-cusolver-cu12==11.7.1.2
130
- triton==3.3.0
131
- pfzy==0.3.4
132
- torchcodec==0.4.0
133
- dm-tree==0.1.8
134
- ninja==1.13.0
135
- nvidia-cufile-cu12==1.11.1.6
136
- datasets==3.6.0
137
- nvidia-curand-cu12==10.3.7.77
138
- flash_attn==2.7.4.post1
139
- av==16.0.1
140
- sniffio==1.3.1
141
- Jinja2==3.1.6
142
- narwhals==2.6.0
143
- yarl==1.20.1
144
- typeguard==4.4.4
145
- termcolor==3.1.0
146
- importlib_metadata==8.7.1
147
- eval_type_backport==0.2.2
148
- certifi==2025.8.3
149
- tzdata==2025.2
150
- typing_extensions==4.15.0
151
- viser==1.0.15
152
- typing-inspect==0.9.0
153
- annotated-types==0.7.0
154
- svg.path==7.0
155
- mergedeep==1.3.4
156
- psi==0.0.0
157
- deepdiff==8.6.1
158
- toml==0.10.2
159
- click==8.3.0
160
- dotenv==0.9.9
161
- trimesh==4.8.3
162
- nvidia-nccl-cu12==2.26.2
163
- pyyaml-include==1.4.1
164
- scikit-image==0.25.2
165
- PySocks==1.7.1
166
- aiohttp==3.10.5
167
- opencv-python==4.11.0.86
168
- nodeenv==1.9.1
169
- lerobot==0.3.3
170
- pillow==11.3.0
171
- idna==3.10
172
- Flask==3.1.3
173
- killport==1.2.0
174
- hf-xet==1.1.10
175
- imageio-ffmpeg==0.6.0
176
- httpcore==1.0.9
177
- nvidia-cusparselt-cu12==0.6.3
178
- networkx==3.4.2
179
- ruamel.yaml==0.18.15
180
- manifold3d==3.2.1
181
- shellingham==1.5.4
182
- nvidia-cufft-cu12==11.3.0.4
183
- mapbox_earcut==1.0.3
184
- peft==0.17.1
185
- pyarrow==21.0.0
186
- pyserial==3.5
187
- torch==2.7.0
188
- dill==0.3.8
189
- markdown-it-py==4.0.0
190
- tqdm==4.67.1
191
- argcomplete==3.6.2
192
- pandas==2.3.3
193
- lxml==6.0.2
194
- safetensors==0.6.2
195
- python-dateutil==2.9.0.post0
196
- regex==2025.9.18
197
- evdev==1.9.3
198
- rich==14.1.0
199
- psi==0.0.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-metadata.json DELETED
@@ -1,132 +0,0 @@
1
- {
2
- "os": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39",
3
- "python": "CPython 3.10.20",
4
- "startedAt": "2026-03-18T14:26:46.537089Z",
5
- "args": [
6
- "simple_act_config",
7
- "--seed=2026",
8
- "--exp=g1wholebodybendpick-v0",
9
- "--train.name=act-g1",
10
- "--log.report-to=wandb",
11
- "--train.data_parallel=ddp",
12
- "--train.mixed_precision=bf16",
13
- "--train.train-batch-size=32",
14
- "--train.warmup-steps=1000",
15
- "--train.warmup-ratio=None",
16
- "--train.checkpointing-steps=5000",
17
- "--train.validation_steps=500",
18
- "--train.val_num_batches=20",
19
- "--train.gradient_accumulation_steps=1",
20
- "--train.max-training-steps=40000",
21
- "--train.learning-rate=1e-4",
22
- "--train.max-grad-norm=1.0",
23
- "--train.lr_scheduler_kwargs.weight_decay=1e-6",
24
- "--train.lr_scheduler_kwargs.betas",
25
- "0.95",
26
- "0.999",
27
- "--train.lr_scheduler_type=cosine",
28
- "--data.root_dir=/data/jliu/data",
29
- "--data.train-repo-ids=G1WholebodyBendPick-v0",
30
- "--data.transform.repack.action_chunk_size=30",
31
- "--data.transform.repack.pad-action-dim=36",
32
- "--data.transform.repack.pad-state-dim=36",
33
- "--data.transform.field.stat-path=meta/stats_psi0.json",
34
- "--data.transform.field.stat-action-key=action",
35
- "--data.transform.field.stat-state-key=states",
36
- "--data.transform.field.normalize-state",
37
- "--data.transform.field.action-norm-type=bounds",
38
- "--data.transform.field.pad-action-dim=36",
39
- "--data.transform.field.pad-state-dim=36",
40
- "--data.transform.model.img-aug",
41
- "--model.chunk-size=30",
42
- "--model.n-action-steps=30",
43
- "--model.action-dim=36",
44
- "--model.state-dim=36",
45
- "--model.use-vae",
46
- "--model.kl-weight=10.0"
47
- ],
48
- "program": "/data/jliu/psi/scripts/train.py",
49
- "codePath": "scripts/train.py",
50
- "codePathLocal": "scripts/train.py",
51
- "git": {
52
- "remote": "https://github.com/songlin/psi.git",
53
- "commit": "04614628ecb677f5f278e2d31c1103aed8127f26"
54
- },
55
- "email": "jliu530@163.com",
56
- "root": "/data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426",
57
- "host": "nebula99",
58
- "executable": "/data/jliu/psi/.venv-psi/bin/python3",
59
- "cpu_count": 128,
60
- "cpu_count_logical": 128,
61
- "gpu": "NVIDIA A100-SXM4-80GB",
62
- "gpu_count": 8,
63
- "disk": {
64
- "/": {
65
- "total": "105089261568",
66
- "used": "99538649088"
67
- }
68
- },
69
- "memory": {
70
- "total": "1623177744384"
71
- },
72
- "gpu_nvidia": [
73
- {
74
- "name": "NVIDIA A100-SXM4-80GB",
75
- "memoryTotal": "85899345920",
76
- "cudaCores": 6912,
77
- "architecture": "Ampere",
78
- "uuid": "GPU-37370f00-90f5-4e03-f763-7a35649e6783"
79
- },
80
- {
81
- "name": "NVIDIA A100-SXM4-80GB",
82
- "memoryTotal": "85899345920",
83
- "cudaCores": 6912,
84
- "architecture": "Ampere",
85
- "uuid": "GPU-d4b40383-188b-cc16-9180-20c1a71a777f"
86
- },
87
- {
88
- "name": "NVIDIA A100-SXM4-80GB",
89
- "memoryTotal": "85899345920",
90
- "cudaCores": 6912,
91
- "architecture": "Ampere",
92
- "uuid": "GPU-0f3d827c-66cf-04da-b182-4ed0414a2549"
93
- },
94
- {
95
- "name": "NVIDIA A100-SXM4-80GB",
96
- "memoryTotal": "85899345920",
97
- "cudaCores": 6912,
98
- "architecture": "Ampere",
99
- "uuid": "GPU-164a49bb-43d3-2250-59e2-1b40eee0757f"
100
- },
101
- {
102
- "name": "NVIDIA A100-SXM4-80GB",
103
- "memoryTotal": "85899345920",
104
- "cudaCores": 6912,
105
- "architecture": "Ampere",
106
- "uuid": "GPU-c0471f40-ae19-5371-a1af-da406ca30f83"
107
- },
108
- {
109
- "name": "NVIDIA A100-SXM4-80GB",
110
- "memoryTotal": "85899345920",
111
- "cudaCores": 6912,
112
- "architecture": "Ampere",
113
- "uuid": "GPU-d9040f59-13d3-6c37-4ea0-c6c751424527"
114
- },
115
- {
116
- "name": "NVIDIA A100-SXM4-80GB",
117
- "memoryTotal": "85899345920",
118
- "cudaCores": 6912,
119
- "architecture": "Ampere",
120
- "uuid": "GPU-f834d0e8-a118-24c7-5461-30fce111bea6"
121
- },
122
- {
123
- "name": "NVIDIA A100-SXM4-80GB",
124
- "memoryTotal": "85899345920",
125
- "cudaCores": 6912,
126
- "architecture": "Ampere",
127
- "uuid": "GPU-6e687915-3247-735c-6641-544540804e79"
128
- }
129
- ],
130
- "cudaVersion": "12.9",
131
- "writerId": "0fepodeqeh00upnrzhqar4dfcf23o49l"
132
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/files/wandb-summary.json DELETED
@@ -1 +0,0 @@
1
- {"val/denorm_err_l1_torso_vyaw":0,"val/denorm_err_l1_rpy":0.0013011818518862128,"val/kld_loss":9.509166702628135e-05,"val/denorm_err_l1_hand_joints":0.0025742806028574705,"_wandb":{"runtime":19620},"train/kld_loss":0.0014440594241023064,"val/bc_loss":0.004661495331674814,"val/denorm_err_l1_torso_vx":0,"val/denorm_err_l1_torso_target_yaw":0,"train/l1_loss":0.0042681763879954815,"train/loss":0.018708771094679832,"_timestamp":1.7738636260787222e+09,"val/denorm_err_l1_height":0.00026183543377555907,"train/epoch":330,"val/denorm_err_l1_arm_joints":0.0026575601659715176,"_step":40000,"val/denorm_err_l1_torso_vy":0,"_runtime":19620.194410914}
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log DELETED
The diff for this file is too large to render. See raw diff
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log DELETED
@@ -1,25 +0,0 @@
1
- 2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
2
- 2026-03-18 14:26:46,542 INFO MainThread:280442 [wandb_setup.py:_flush():81] Configure stats pid to 280442
3
- 2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
- 2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug.log
5
- 2026-03-18 14:26:46,543 INFO MainThread:280442 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/logs/debug-internal.log
6
- 2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():844] calling init triggers
7
- 2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
- config: {'_wandb': {}}
9
- 2026-03-18 14:26:46,544 INFO MainThread:280442 [wandb_init.py:init():892] starting backend
10
- 2026-03-18 14:26:46,802 INFO MainThread:280442 [wandb_init.py:init():895] sending inform_init request
11
- 2026-03-18 14:26:46,814 INFO MainThread:280442 [wandb_init.py:init():903] backend started and connected
12
- 2026-03-18 14:26:46,815 INFO MainThread:280442 [wandb_init.py:init():973] updated telemetry
13
- 2026-03-18 14:26:46,822 INFO MainThread:280442 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
- 2026-03-18 14:26:48,282 INFO MainThread:280442 [wandb_init.py:init():1042] starting run threads in backend
15
- 2026-03-18 14:26:48,444 INFO MainThread:280442 [wandb_run.py:_console_start():2524] atexit reg
16
- 2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
- 2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
- 2026-03-18 14:26:48,445 INFO MainThread:280442 [wandb_run.py:_redirect():2465] Redirects installed.
19
- 2026-03-18 14:26:48,451 INFO MainThread:280442 [wandb_init.py:init():1082] run started, returning control to user process
20
- 2026-03-18 14:26:48,454 INFO MainThread:280442 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodybendpick-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2603181426', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.13059291243553162, -0.09108058363199234, -0.0024844733998179436, -0.20733775198459625, -0.15850023925304413, -0.17450474202632904, -0.2997315526008606, -0.015391111373901367, -0.34571564197540283, -0.4991437792778015, 0.0, 0.0, 0.0, 0.0, -0.1015840545296669, -0.06647031009197235, -0.16578954458236694, -0.14477218687534332, -0.3665394186973572, -0.28364259004592896, -0.1775387078523636, -0.48419490456581116, -0.7551082968711853, -0.2692946195602417, -0.03164339065551758, -3.876500704791397e-05, -0.3909206688404083, 0.0, -0.04351663216948509, -0.014203650876879692, -0.049649015069007874, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.08620641380548477, 0.13058121502399445, 0.22948147356510162, 0.020551620051264763, 0.005824880674481392, 0.010019193403422832, 8.43817247186962e-07, 0.39566752314567566, 0.0, 0.0, 0.4860266447067261, 1.0467392206192017, 0.6470075845718384, 0.8298009037971497, 0.03516175225377083, 0.11019192636013031, 0.04779902100563049, 0.12850724160671234, 3.8688118365826085e-05, 0.0012142359046265483, 3.3343669201713055e-05, 0.002679983852431178, 0.00041063950629904866, 0.1973484456539154, 0.2633756697177887, 0.34943076968193054, 0.0012102096807211637, 0.8342975974082947, 0.31870752573013306, 0.45533719658851624, 0.15729404985904694, 0.75, 0.0, 0.0, 0.0, 0.0], 'state_min': [-0.13899999856948853, -0.09099991619586945, -5.989517215532203e-11, -0.20900000631809235, -0.1589999943971634, -0.210999995470047, -0.3009999990463257, -0.01600000075995922, -0.1860000044107437, -0.6940000057220459, 0.0, 0.0, 0.0, 0.0, -0.1019991859793663, -0.06899992376565933, -0.16899999976158142, -0.14499999582767487, -0.3709999918937683, -0.28700000047683716, -0.17800045013427734, -0.4869999885559082, -0.7599999904632568, -0.27300000190734863, -0.029999999329447746, 0.0, -0.39100033044815063, -0.0010000000474974513, 0.0, -0.15000000596046448, 0.0, 0.44999998807907104, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.0860000029206276, 0.2720000147819519, 0.23100000619888306, 0.0, 6.510182259944486e-08, 0.0, 0.0, 0.5550000071525574, 0.02100004442036152, 0.0, 0.5429999828338623, 1.13100004196167, 0.5770000219345093, 0.9580000042915344, 0.07141251862049103, 0.10899999737739563, 0.04699999839067459, 0.13600000739097595, 0.0, 0.003000000026077032, 0.0, 0.009999999776482582, 0.0020000000949949026, 0.2029999941587448, 0.2759999930858612, 0.3499999940395355, 0.003000000026077032, 0.8370000123977661, 0.0, 0.0, 0.0, 0.75, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyBendPick-v0'], 'val_repo_ids': ['G1WholebodyBendPick-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '0,1,2,3', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '37919'}}
21
- 2026-03-18 19:53:48,477 INFO MainThread:280442 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/agq65opl
22
- 2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
23
- 2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2472] restore
24
- 2026-03-18 19:53:48,479 INFO MainThread:280442 [wandb_run.py:_restore():2478] restore done
25
- 2026-03-18 19:53:50,040 INFO MainThread:280442 [wandb_run.py:_footer_sync_info():3868] logging synced files
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodybendpick-v0.g1.cosine.lr1.0e-04.b128.gpus4.2603181426/wandb/run-20260318_142646-agq65opl/run-agq65opl.wandb DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c4763659b36203ca292aadbe415653bf031cd65a551620a19105912bae817a97
3
- size 52535712
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/argv.txt DELETED
@@ -1,40 +0,0 @@
1
- scripts/train.py
2
- simple_act_config
3
- --seed=2026
4
- --exp=g1wholebodyhandover-v0
5
- --train.name=act-g1
6
- --log.report-to=wandb
7
- --train.data_parallel=ddp
8
- --train.mixed_precision=bf16
9
- --train.train-batch-size=32
10
- --train.warmup-steps=1000
11
- --train.warmup-ratio=None
12
- --train.checkpointing-steps=5000
13
- --train.validation_steps=500
14
- --train.val_num_batches=20
15
- --train.gradient_accumulation_steps=1
16
- --train.max-training-steps=40000
17
- --train.learning-rate=1e-4
18
- --train.max-grad-norm=1.0
19
- --train.lr_scheduler_kwargs.weight_decay=1e-6
20
- --train.lr_scheduler_kwargs.betas 0.95 0.999
21
- --train.lr_scheduler_type=cosine
22
- --data.root_dir=/data/jliu/data
23
- --data.train-repo-ids=G1WholebodyHandover-v0
24
- --data.transform.repack.action_chunk_size=30
25
- --data.transform.repack.pad-action-dim=36
26
- --data.transform.repack.pad-state-dim=36
27
- --data.transform.field.stat-path=meta/stats_psi0.json
28
- --data.transform.field.stat-action-key=action
29
- --data.transform.field.stat-state-key=states
30
- --data.transform.field.normalize-state
31
- --data.transform.field.action-norm-type=bounds
32
- --data.transform.field.pad-action-dim=36
33
- --data.transform.field.pad-state-dim=36
34
- --data.transform.model.img-aug
35
- --model.chunk-size=30
36
- --model.n-action-steps=30
37
- --model.action-dim=36
38
- --model.state-dim=36
39
- --model.use-vae
40
- --model.kl-weight=10.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:be3cee501e8e37aa7dc99358fb114794cfe604e00da6ae82e00ec88a9b429c8a
3
- size 206658952
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/optimizer.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b7dc327459001762d0a2f688167d42a05a461410278eb668a60bcd532f829d01
3
- size 413111371
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_0.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:debdcafc5d326bc0876e3268ce9be296d94f9cfd67a489f44ee827cacd1c0699
3
- size 15473
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_1.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:08b5ccfa2fc4720532d3399cfb5bff7d3be670aa7790b0de564ede016cb0c973
3
- size 15473
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_2.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:43b76b6dcb5c2116e0217a758aadd5240580ffa1cb0873287aaef42984ccad10
3
- size 15473
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/scheduler.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:edd2ef4de5774723b53b76baced1d134ab2a534e3951c7a3ebfbc157468c8f72
3
- size 1401
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/envs.txt DELETED
@@ -1,17 +0,0 @@
1
- OMP_NUM_THREADS=32
2
- HF_HOME=/data/cache
3
- TORCH_HOME=/data/cache
4
- HF_TOKEN=hf_...TiKa
5
- HF_LEROBOT_HOME=/data/data/lerobot
6
- WE_HOME=Not Set
7
- DATA_HOME=/data/data
8
- UV_CACHE_DIR=/data/cache
9
- WANDB_API_KEY=90e...5c06
10
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
11
- CUDA_VISIBLE_DEVICES=4,5,6,7
12
- WORLD_SIZE=4
13
- LOCAL_WORLD_SIZE=4
14
- RANK=0
15
- LOCAL_RANK=0
16
- MASTER_ADDR=nebula99
17
- MASTER_PORT=40557
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/run_config.json DELETED
@@ -1,310 +0,0 @@
1
- {
2
- "exp": "g1wholebodyhandover-v0",
3
- "seed": 2026,
4
- "auto_tag_run": false,
5
- "eval": false,
6
- "debug": false,
7
- "timestamp": "2604071503",
8
- "log": {
9
- "logging_dir": "logs",
10
- "report_to": "wandb",
11
- "log_freq": 100
12
- },
13
- "wandb": {
14
- "project": "psi",
15
- "entity": "jliu530-soochow-university",
16
- "group": "act-g1",
17
- "id": "26s2ml9t",
18
- "name": "g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503",
19
- "resume": "allow"
20
- },
21
- "train": {
22
- "num_workers": 8,
23
- "overfit_single_batch": false,
24
- "name": "act-g1",
25
- "resume_from_checkpoint": null,
26
- "skip_resumed_steps": false,
27
- "hf_token": ".hf_token",
28
- "lora": false,
29
- "output_dir": ".runs",
30
- "gradient_accumulation_steps": 1,
31
- "mixed_precision": "bf16",
32
- "max_grad_norm": 1.0,
33
- "optimizer_foreach": null,
34
- "train_batch_size": 32,
35
- "val_batch_size": 16,
36
- "val_num_batches": 20,
37
- "checkpointing_steps": 5000,
38
- "max_checkpoints_to_keep": null,
39
- "validation_steps": 500,
40
- "learning_rate": 0.0001,
41
- "lr_scheduler_type": "cosine",
42
- "lr_scheduler_kwargs": {
43
- "betas": [
44
- 0.95,
45
- 0.999
46
- ],
47
- "weight_decay": 1e-6,
48
- "eps": 1e-8
49
- },
50
- "scheduler_specific_kwargs": {},
51
- "data_parallel": "ddp",
52
- "sharding_strategy": "full-shard",
53
- "deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
54
- "enable_gradient_checkpointing": true,
55
- "enable_mixed_precision_training": true,
56
- "reduce_in_full_precision": true,
57
- "max_training_steps": 40000,
58
- "num_train_epochs": null,
59
- "warmup_steps": 1000,
60
- "warmup_ratio": null
61
- },
62
- "data": {
63
- "transform": {
64
- "repack": {
65
- "dataset_name": "simple",
66
- "num_past_frames": 0,
67
- "action_chunk_size": 30,
68
- "pad_action_dim": 36,
69
- "pad_state_dim": 36
70
- },
71
- "model": {
72
- "resize": {
73
- "size": [
74
- 256,
75
- 480
76
- ]
77
- },
78
- "center_crop": {
79
- "size": [
80
- 224,
81
- 224
82
- ]
83
- },
84
- "color_jitter": {
85
- "brightness": 0.2,
86
- "contrast": [
87
- 0.8,
88
- 1.2
89
- ],
90
- "saturation": [
91
- 0.8,
92
- 1.2
93
- ],
94
- "hue": 0.05
95
- },
96
- "normalize": {
97
- "mean": [
98
- 0.485,
99
- 0.456,
100
- 0.406
101
- ],
102
- "std": [
103
- 0.229,
104
- 0.224,
105
- 0.225
106
- ]
107
- },
108
- "img_aug": true
109
- },
110
- "field": {
111
- "stat_path": "meta/stats_psi0.json",
112
- "action_norm_type": "bounds",
113
- "stat_action_key": "action",
114
- "stat_state_key": "states",
115
- "use_norm_mask": false,
116
- "action_norm_masks": [
117
- true,
118
- true,
119
- true,
120
- true,
121
- true,
122
- true,
123
- false
124
- ],
125
- "action_min": [
126
- -0.5,
127
- -1.1095792134107943e-16,
128
- -1.1095792134107943e-16,
129
- -1.5,
130
- -1.5,
131
- -1.5,
132
- -1.5,
133
- -0.5,
134
- -0.699999988079071,
135
- -0.699999988079071,
136
- -2.2166350627321588e-16,
137
- -2.2166350627321588e-16,
138
- 0.0,
139
- -2.2166350627321588e-16,
140
- -0.47567468881607056,
141
- 0.1900009959936142,
142
- -0.512170135974884,
143
- -0.6265152096748352,
144
- -0.5008617043495178,
145
- -0.8220608830451965,
146
- -0.9223371148109436,
147
- -0.49507391452789307,
148
- -0.3437551259994507,
149
- -0.6871383190155029,
150
- -0.7637607455253601,
151
- -0.7568023204803467,
152
- -0.576077401638031,
153
- -0.4588268995285034,
154
- -0.13876836001873016,
155
- -0.10360867530107498,
156
- -0.47856518626213074,
157
- 0.7400000095367432,
158
- -0.5,
159
- -0.5,
160
- -0.26161932945251465,
161
- -0.06718750298023224
162
- ],
163
- "action_max": [
164
- 0.5,
165
- 0.699999988079071,
166
- 0.699999988079071,
167
- 0.0,
168
- 2.2146225653890418e-16,
169
- 2.2146225653890418e-16,
170
- 2.2146225653890418e-16,
171
- 1.2266071310501902e-19,
172
- 1.1078670818917075e-16,
173
- 1.1078670818917075e-16,
174
- 1.5,
175
- 1.5,
176
- 0.6000000238418579,
177
- 1.5,
178
- 0.2472410947084427,
179
- 0.7092280983924866,
180
- 1.2571598291397095,
181
- 0.42311304807662964,
182
- 0.8564174771308899,
183
- 0.5002086162567139,
184
- 0.5172277092933655,
185
- 0.16140148043632507,
186
- -0.1900009959936142,
187
- 0.5362864136695862,
188
- 0.5715147256851196,
189
- 0.5002322196960449,
190
- 0.566592276096344,
191
- 0.6392397880554199,
192
- 0.1580466777086258,
193
- 0.2233395129442215,
194
- 0.2582152187824249,
195
- 0.7400000095367432,
196
- 0.5,
197
- 0.5,
198
- 0.3454970121383667,
199
- 0.2899305522441864
200
- ],
201
- "state_min": [
202
- -0.5564982891082764,
203
- -0.48307520151138306,
204
- -0.0005447770818136632,
205
- -0.8388738632202148,
206
- -1.3970016241073608,
207
- -0.8296014666557312,
208
- -1.4599460363388062,
209
- -0.5806806683540344,
210
- -0.5149835348129272,
211
- -0.6775947213172913,
212
- -0.001480442238971591,
213
- -0.0002713006397243589,
214
- -0.000914653530344367,
215
- -0.00019419840828049928,
216
- -0.4206617772579193,
217
- 0.13972464203834534,
218
- -0.546251654624939,
219
- -0.5596316456794739,
220
- -0.4764360189437866,
221
- -0.7253566384315491,
222
- -0.9443663954734802,
223
- -0.4381798803806305,
224
- -0.3338131606578827,
225
- -0.667724072933197,
226
- -0.6881827116012573,
227
- -0.7544379830360413,
228
- -0.5189417600631714,
229
- -0.4484957158565521,
230
- -0.13709338009357452,
231
- -0.07360810041427612,
232
- -0.4748336970806122,
233
- 0.7400000095367432,
234
- 0.0,
235
- 0.0,
236
- 0.0,
237
- 0.0
238
- ],
239
- "state_max": [
240
- 0.43566983938217163,
241
- 0.3739710748195648,
242
- 0.6575677990913391,
243
- 0.004060761071741581,
244
- 0.0005700877518393099,
245
- 0.0004725759499706328,
246
- 0.00010080631182063371,
247
- 0.00001310737025050912,
248
- 0.21882089972496033,
249
- 0.0005271440604701638,
250
- 0.530737042427063,
251
- 1.4406861066818237,
252
- 1.4605127573013306,
253
- 1.4595911502838135,
254
- 0.2663630545139313,
255
- 0.657910943031311,
256
- 1.2515853643417358,
257
- 0.502498209476471,
258
- 0.8292973637580872,
259
- 0.5248894095420837,
260
- 0.4653257131576538,
261
- 0.18638382852077484,
262
- -0.16696421802043915,
263
- 0.49318820238113403,
264
- 0.6363148093223572,
265
- 0.45773962140083313,
266
- 0.6238265037536621,
267
- 0.653800904750824,
268
- 0.1436084657907486,
269
- 0.25937986373901367,
270
- 0.26422709226608276,
271
- 0.7400000095367432,
272
- 0.0,
273
- 0.0,
274
- 0.0,
275
- 0.0
276
- ],
277
- "normalize_state": true,
278
- "pad_action_dim": 36,
279
- "pad_state_dim": 36
280
- }
281
- },
282
- "root_dir": "/data/jliu/data",
283
- "train_repo_ids": [
284
- "G1WholebodyHandover-v0"
285
- ],
286
- "val_repo_ids": [
287
- "G1WholebodyHandover-v0"
288
- ]
289
- },
290
- "model": {
291
- "n_obs_steps": 1,
292
- "chunk_size": 30,
293
- "n_action_steps": 30,
294
- "action_dim": 36,
295
- "state_dim": 36,
296
- "dim_model": 512,
297
- "n_heads": 8,
298
- "dim_feedforward": 3200,
299
- "feedforward_activation": "relu",
300
- "n_encoder_layers": 4,
301
- "n_decoder_layers": 1,
302
- "pre_norm": false,
303
- "dropout": 0.1,
304
- "use_vae": true,
305
- "latent_dim": 32,
306
- "n_vae_encoder_layers": 4,
307
- "kl_weight": 10.0,
308
- "temporal_ensemble_coeff": null
309
- }
310
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug-internal.log DELETED
The diff for this file is too large to render. See raw diff
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/debug.log DELETED
@@ -1,25 +0,0 @@
1
- 2026-04-07 15:03:14,310 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
2
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Configure stats pid to 2236517
3
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log
5
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log
6
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():844] calling init triggers
7
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
- config: {'_wandb': {}}
9
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():892] starting backend
10
- 2026-04-07 15:03:14,567 INFO MainThread:2236517 [wandb_init.py:init():895] sending inform_init request
11
- 2026-04-07 15:03:14,577 INFO MainThread:2236517 [wandb_init.py:init():903] backend started and connected
12
- 2026-04-07 15:03:14,578 INFO MainThread:2236517 [wandb_init.py:init():973] updated telemetry
13
- 2026-04-07 15:03:14,584 INFO MainThread:2236517 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
- 2026-04-07 15:03:15,461 INFO MainThread:2236517 [wandb_init.py:init():1042] starting run threads in backend
15
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_console_start():2524] atexit reg
16
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2465] Redirects installed.
19
- 2026-04-07 15:03:15,625 INFO MainThread:2236517 [wandb_init.py:init():1082] run started, returning control to user process
20
- 2026-04-07 15:03:15,626 INFO MainThread:2236517 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodyhandover-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604071503', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.5, -1.1095792134107943e-16, -1.1095792134107943e-16, -1.5, -1.5, -1.5, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.2166350627321588e-16, -2.2166350627321588e-16, 0.0, -2.2166350627321588e-16, -0.47567468881607056, 0.1900009959936142, -0.512170135974884, -0.6265152096748352, -0.5008617043495178, -0.8220608830451965, -0.9223371148109436, -0.49507391452789307, -0.3437551259994507, -0.6871383190155029, -0.7637607455253601, -0.7568023204803467, -0.576077401638031, -0.4588268995285034, -0.13876836001873016, -0.10360867530107498, -0.47856518626213074, 0.7400000095367432, -0.5, -0.5, -0.26161932945251465, -0.06718750298023224], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 2.2146225653890418e-16, 2.2146225653890418e-16, 2.2146225653890418e-16, 1.2266071310501902e-19, 1.1078670818917075e-16, 1.1078670818917075e-16, 1.5, 1.5, 0.6000000238418579, 1.5, 0.2472410947084427, 0.7092280983924866, 1.2571598291397095, 0.42311304807662964, 0.8564174771308899, 0.5002086162567139, 0.5172277092933655, 0.16140148043632507, -0.1900009959936142, 0.5362864136695862, 0.5715147256851196, 0.5002322196960449, 0.566592276096344, 0.6392397880554199, 0.1580466777086258, 0.2233395129442215, 0.2582152187824249, 0.7400000095367432, 0.5, 0.5, 0.3454970121383667, 0.2899305522441864], 'state_min': [-0.5564982891082764, -0.48307520151138306, -0.0005447770818136632, -0.8388738632202148, -1.3970016241073608, -0.8296014666557312, -1.4599460363388062, -0.5806806683540344, -0.5149835348129272, -0.6775947213172913, -0.001480442238971591, -0.0002713006397243589, -0.000914653530344367, -0.00019419840828049928, -0.4206617772579193, 0.13972464203834534, -0.546251654624939, -0.5596316456794739, -0.4764360189437866, -0.7253566384315491, -0.9443663954734802, -0.4381798803806305, -0.3338131606578827, -0.667724072933197, -0.6881827116012573, -0.7544379830360413, -0.5189417600631714, -0.4484957158565521, -0.13709338009357452, -0.07360810041427612, -0.4748336970806122, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.43566983938217163, 0.3739710748195648, 0.6575677990913391, 0.004060761071741581, 0.0005700877518393099, 0.0004725759499706328, 0.00010080631182063371, 1.310737025050912e-05, 0.21882089972496033, 0.0005271440604701638, 0.530737042427063, 1.4406861066818237, 1.4605127573013306, 1.4595911502838135, 0.2663630545139313, 0.657910943031311, 1.2515853643417358, 0.502498209476471, 0.8292973637580872, 0.5248894095420837, 0.4653257131576538, 0.18638382852077484, -0.16696421802043915, 0.49318820238113403, 0.6363148093223572, 0.45773962140083313, 0.6238265037536621, 0.653800904750824, 0.1436084657907486, 0.25937986373901367, 0.26422709226608276, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyHandover-v0'], 'val_repo_ids': ['G1WholebodyHandover-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '40557'}}
21
- 2026-04-07 18:06:44,913 INFO MainThread:2236517 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/26s2ml9t
22
- 2026-04-07 18:06:44,914 INFO MainThread:2236517 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
23
- 2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2472] restore
24
- 2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2478] restore done
25
- 2026-04-07 18:06:45,754 INFO MainThread:2236517 [wandb_run.py:_footer_sync_info():3868] logging synced files
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/config.yaml DELETED
@@ -1,448 +0,0 @@
1
- _wandb:
2
- value:
3
- cli_version: 0.25.1
4
- e:
5
- 1ney3xhb33gta4c2m7gfcmat7y4xd0vk:
6
- args:
7
- - simple_act_config
8
- - --seed=2026
9
- - --exp=g1wholebodyhandover-v0
10
- - --train.name=act-g1
11
- - --log.report-to=wandb
12
- - --train.data_parallel=ddp
13
- - --train.mixed_precision=bf16
14
- - --train.train-batch-size=32
15
- - --train.warmup-steps=1000
16
- - --train.warmup-ratio=None
17
- - --train.checkpointing-steps=5000
18
- - --train.validation_steps=500
19
- - --train.val_num_batches=20
20
- - --train.gradient_accumulation_steps=1
21
- - --train.max-training-steps=40000
22
- - --train.learning-rate=1e-4
23
- - --train.max-grad-norm=1.0
24
- - --train.lr_scheduler_kwargs.weight_decay=1e-6
25
- - --train.lr_scheduler_kwargs.betas
26
- - "0.95"
27
- - "0.999"
28
- - --train.lr_scheduler_type=cosine
29
- - --data.root_dir=/data/jliu/data
30
- - --data.train-repo-ids=G1WholebodyHandover-v0
31
- - --data.transform.repack.action_chunk_size=30
32
- - --data.transform.repack.pad-action-dim=36
33
- - --data.transform.repack.pad-state-dim=36
34
- - --data.transform.field.stat-path=meta/stats_psi0.json
35
- - --data.transform.field.stat-action-key=action
36
- - --data.transform.field.stat-state-key=states
37
- - --data.transform.field.normalize-state
38
- - --data.transform.field.action-norm-type=bounds
39
- - --data.transform.field.pad-action-dim=36
40
- - --data.transform.field.pad-state-dim=36
41
- - --data.transform.model.img-aug
42
- - --model.chunk-size=30
43
- - --model.n-action-steps=30
44
- - --model.action-dim=36
45
- - --model.state-dim=36
46
- - --model.use-vae
47
- - --model.kl-weight=10.0
48
- codePath: scripts/train.py
49
- codePathLocal: scripts/train.py
50
- cpu_count: 128
51
- cpu_count_logical: 128
52
- cudaVersion: "12.9"
53
- disk:
54
- /:
55
- total: "105089261568"
56
- used: "89297772544"
57
- email: jliu530@163.com
58
- executable: /data/jliu/psi/.venv-psi/bin/python3
59
- git:
60
- commit: 04614628ecb677f5f278e2d31c1103aed8127f26
61
- remote: https://github.com/songlin/psi.git
62
- gpu: NVIDIA A100-SXM4-80GB
63
- gpu_count: 8
64
- gpu_nvidia:
65
- - architecture: Ampere
66
- cudaCores: 6912
67
- memoryTotal: "85899345920"
68
- name: NVIDIA A100-SXM4-80GB
69
- uuid: GPU-02b51758-2aec-8c71-c751-92567fdb15ad
70
- - architecture: Ampere
71
- cudaCores: 6912
72
- memoryTotal: "85899345920"
73
- name: NVIDIA A100-SXM4-80GB
74
- uuid: GPU-be645bb4-1ef0-15da-fc6b-53c5378650df
75
- - architecture: Ampere
76
- cudaCores: 6912
77
- memoryTotal: "85899345920"
78
- name: NVIDIA A100-SXM4-80GB
79
- uuid: GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1
80
- - architecture: Ampere
81
- cudaCores: 6912
82
- memoryTotal: "85899345920"
83
- name: NVIDIA A100-SXM4-80GB
84
- uuid: GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b
85
- - architecture: Ampere
86
- cudaCores: 6912
87
- memoryTotal: "85899345920"
88
- name: NVIDIA A100-SXM4-80GB
89
- uuid: GPU-57787183-4904-b2c9-cf52-1c1af976b365
90
- - architecture: Ampere
91
- cudaCores: 6912
92
- memoryTotal: "85899345920"
93
- name: NVIDIA A100-SXM4-80GB
94
- uuid: GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3
95
- - architecture: Ampere
96
- cudaCores: 6912
97
- memoryTotal: "85899345920"
98
- name: NVIDIA A100-SXM4-80GB
99
- uuid: GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c
100
- - architecture: Ampere
101
- cudaCores: 6912
102
- memoryTotal: "85899345920"
103
- name: NVIDIA A100-SXM4-80GB
104
- uuid: GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8
105
- host: nebula99
106
- memory:
107
- total: "1623177445376"
108
- os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
109
- program: /data/jliu/psi/scripts/train.py
110
- python: CPython 3.10.20
111
- root: /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503
112
- startedAt: "2026-04-07T15:03:14.309147Z"
113
- writerId: 1ney3xhb33gta4c2m7gfcmat7y4xd0vk
114
- m: []
115
- python_version: 3.10.20
116
- t:
117
- "1":
118
- - 1
119
- - 11
120
- - 41
121
- - 49
122
- - 71
123
- "2":
124
- - 1
125
- - 11
126
- - 41
127
- - 49
128
- - 51
129
- - 71
130
- - 83
131
- "3":
132
- - 2
133
- - 13
134
- - 61
135
- "4": 3.10.20
136
- "5": 0.25.1
137
- "6": 4.57.0
138
- "12": 0.25.1
139
- "13": linux-x86_64
140
- auto_tag_run:
141
- value: false
142
- data:
143
- value:
144
- root_dir: /data/jliu/data
145
- train_repo_ids:
146
- - G1WholebodyHandover-v0
147
- transform:
148
- field:
149
- action_max:
150
- - 0.5
151
- - 0.699999988079071
152
- - 0.699999988079071
153
- - 0
154
- - 2.2146225653890418e-16
155
- - 2.2146225653890418e-16
156
- - 2.2146225653890418e-16
157
- - 1.2266071310501902e-19
158
- - 1.1078670818917075e-16
159
- - 1.1078670818917075e-16
160
- - 1.5
161
- - 1.5
162
- - 0.6000000238418579
163
- - 1.5
164
- - 0.2472410947084427
165
- - 0.7092280983924866
166
- - 1.2571598291397095
167
- - 0.42311304807662964
168
- - 0.8564174771308899
169
- - 0.5002086162567139
170
- - 0.5172277092933655
171
- - 0.16140148043632507
172
- - -0.1900009959936142
173
- - 0.5362864136695862
174
- - 0.5715147256851196
175
- - 0.5002322196960449
176
- - 0.566592276096344
177
- - 0.6392397880554199
178
- - 0.1580466777086258
179
- - 0.2233395129442215
180
- - 0.2582152187824249
181
- - 0.7400000095367432
182
- - 0.5
183
- - 0.5
184
- - 0.3454970121383667
185
- - 0.2899305522441864
186
- action_min:
187
- - -0.5
188
- - -1.1095792134107943e-16
189
- - -1.1095792134107943e-16
190
- - -1.5
191
- - -1.5
192
- - -1.5
193
- - -1.5
194
- - -0.5
195
- - -0.699999988079071
196
- - -0.699999988079071
197
- - -2.2166350627321588e-16
198
- - -2.2166350627321588e-16
199
- - 0
200
- - -2.2166350627321588e-16
201
- - -0.47567468881607056
202
- - 0.1900009959936142
203
- - -0.512170135974884
204
- - -0.6265152096748352
205
- - -0.5008617043495178
206
- - -0.8220608830451965
207
- - -0.9223371148109436
208
- - -0.49507391452789307
209
- - -0.3437551259994507
210
- - -0.6871383190155029
211
- - -0.7637607455253601
212
- - -0.7568023204803467
213
- - -0.576077401638031
214
- - -0.4588268995285034
215
- - -0.13876836001873016
216
- - -0.10360867530107498
217
- - -0.47856518626213074
218
- - 0.7400000095367432
219
- - -0.5
220
- - -0.5
221
- - -0.26161932945251465
222
- - -0.06718750298023224
223
- action_norm_masks:
224
- - true
225
- - true
226
- - true
227
- - true
228
- - true
229
- - true
230
- - false
231
- action_norm_type: bounds
232
- normalize_state: true
233
- pad_action_dim: 36
234
- pad_state_dim: 36
235
- stat_action_key: action
236
- stat_path: meta/stats_psi0.json
237
- stat_state_key: states
238
- state_max:
239
- - 0.43566983938217163
240
- - 0.3739710748195648
241
- - 0.6575677990913391
242
- - 0.004060761071741581
243
- - 0.0005700877518393099
244
- - 0.0004725759499706328
245
- - 0.00010080631182063371
246
- - 1.310737025050912e-05
247
- - 0.21882089972496033
248
- - 0.0005271440604701638
249
- - 0.530737042427063
250
- - 1.4406861066818237
251
- - 1.4605127573013306
252
- - 1.4595911502838135
253
- - 0.2663630545139313
254
- - 0.657910943031311
255
- - 1.2515853643417358
256
- - 0.502498209476471
257
- - 0.8292973637580872
258
- - 0.5248894095420837
259
- - 0.4653257131576538
260
- - 0.18638382852077484
261
- - -0.16696421802043915
262
- - 0.49318820238113403
263
- - 0.6363148093223572
264
- - 0.45773962140083313
265
- - 0.6238265037536621
266
- - 0.653800904750824
267
- - 0.1436084657907486
268
- - 0.25937986373901367
269
- - 0.26422709226608276
270
- - 0.7400000095367432
271
- - 0
272
- - 0
273
- - 0
274
- - 0
275
- state_min:
276
- - -0.5564982891082764
277
- - -0.48307520151138306
278
- - -0.0005447770818136632
279
- - -0.8388738632202148
280
- - -1.3970016241073608
281
- - -0.8296014666557312
282
- - -1.4599460363388062
283
- - -0.5806806683540344
284
- - -0.5149835348129272
285
- - -0.6775947213172913
286
- - -0.001480442238971591
287
- - -0.0002713006397243589
288
- - -0.000914653530344367
289
- - -0.00019419840828049928
290
- - -0.4206617772579193
291
- - 0.13972464203834534
292
- - -0.546251654624939
293
- - -0.5596316456794739
294
- - -0.4764360189437866
295
- - -0.7253566384315491
296
- - -0.9443663954734802
297
- - -0.4381798803806305
298
- - -0.3338131606578827
299
- - -0.667724072933197
300
- - -0.6881827116012573
301
- - -0.7544379830360413
302
- - -0.5189417600631714
303
- - -0.4484957158565521
304
- - -0.13709338009357452
305
- - -0.07360810041427612
306
- - -0.4748336970806122
307
- - 0.7400000095367432
308
- - 0
309
- - 0
310
- - 0
311
- - 0
312
- use_norm_mask: false
313
- model:
314
- center_crop:
315
- size:
316
- - 224
317
- - 224
318
- color_jitter:
319
- brightness: 0.2
320
- contrast:
321
- - 0.8
322
- - 1.2
323
- hue: 0.05
324
- saturation:
325
- - 0.8
326
- - 1.2
327
- img_aug: true
328
- normalize:
329
- mean:
330
- - 0.485
331
- - 0.456
332
- - 0.406
333
- std:
334
- - 0.229
335
- - 0.224
336
- - 0.225
337
- resize:
338
- size:
339
- - 256
340
- - 480
341
- repack:
342
- action_chunk_size: 30
343
- dataset_name: simple
344
- num_past_frames: 0
345
- pad_action_dim: 36
346
- pad_state_dim: 36
347
- val_repo_ids:
348
- - G1WholebodyHandover-v0
349
- debug:
350
- value: false
351
- environment_variables:
352
- value:
353
- CUDA_VISIBLE_DEVICES: 4,5,6,7
354
- DATA_HOME: /data/data
355
- HF_HOME: /data/cache
356
- HF_LEROBOT_HOME: /data/data/lerobot
357
- HF_TOKEN: hf_...TiKa
358
- LOCAL_RANK: "0"
359
- LOCAL_WORLD_SIZE: "4"
360
- MASTER_ADDR: nebula99
361
- MASTER_PORT: "40557"
362
- OMP_NUM_THREADS: "32"
363
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
364
- RANK: "0"
365
- TORCH_HOME: /data/cache
366
- UV_CACHE_DIR: /data/cache
367
- WANDB_API_KEY: 90e...5c06
368
- WE_HOME: Not Set
369
- WORLD_SIZE: "4"
370
- eval:
371
- value: false
372
- exp:
373
- value: g1wholebodyhandover-v0
374
- log:
375
- value:
376
- log_freq: 100
377
- logging_dir: logs
378
- report_to: wandb
379
- model:
380
- value:
381
- action_dim: 36
382
- chunk_size: 30
383
- dim_feedforward: 3200
384
- dim_model: 512
385
- dropout: 0.1
386
- feedforward_activation: relu
387
- kl_weight: 10
388
- latent_dim: 32
389
- n_action_steps: 30
390
- n_decoder_layers: 1
391
- n_encoder_layers: 4
392
- n_heads: 8
393
- n_obs_steps: 1
394
- n_vae_encoder_layers: 4
395
- pre_norm: false
396
- state_dim: 36
397
- temporal_ensemble_coeff: null
398
- use_vae: true
399
- seed:
400
- value: 2026
401
- timestamp:
402
- value: "2604071503"
403
- train:
404
- value:
405
- checkpointing_steps: 5000
406
- data_parallel: ddp
407
- deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
408
- enable_gradient_checkpointing: true
409
- enable_mixed_precision_training: true
410
- gradient_accumulation_steps: 1
411
- hf_token: .hf_token
412
- learning_rate: 0.0001
413
- lora: false
414
- lr_scheduler_kwargs:
415
- betas:
416
- - 0.95
417
- - 0.999
418
- eps: 1e-08
419
- weight_decay: 1e-06
420
- lr_scheduler_type: cosine
421
- max_checkpoints_to_keep: null
422
- max_grad_norm: 1
423
- max_training_steps: 40000
424
- mixed_precision: bf16
425
- name: act-g1
426
- num_train_epochs: null
427
- num_workers: 8
428
- optimizer_foreach: null
429
- output_dir: .runs
430
- overfit_single_batch: false
431
- reduce_in_full_precision: true
432
- resume_from_checkpoint: null
433
- sharding_strategy: full-shard
434
- skip_resumed_steps: false
435
- train_batch_size: 32
436
- val_batch_size: 16
437
- val_num_batches: 20
438
- validation_steps: 500
439
- warmup_ratio: null
440
- warmup_steps: 1000
441
- wandb:
442
- value:
443
- entity: jliu530-soochow-university
444
- group: null
445
- id: null
446
- name: null
447
- project: psi
448
- resume: allow
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/output.log DELETED
@@ -1,179 +0,0 @@
1
- [15:03:15 04/07] INFO  | >> [*] Saved configuration to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503 ]8;id=805427;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\128]8;;\
2
-   INFO  | >> [*] Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\181]8;;\
3
-   INFO  | >> |=> training task: 'act-g1' ]8;id=471497;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\182]8;;\
4
-   INFO  | >> |=> run name: g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503 ]8;id=329497;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\183]8;;\
5
-   INFO  | >> |=> seed: 2026 ]8;id=844246;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\184]8;;\
6
-   INFO  | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\185]8;;\
7
-   INFO  | >> |=> warmup steps: 1000 ]8;id=147732;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\186]8;;\
8
-   INFO  | >> |=> validation steps: 500 ]8;id=25941;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\187]8;;\
9
-   INFO  | >> |=> checkpoint steps: 5000 ]8;id=95089;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\188]8;;\
10
-   INFO  | >> |=> max gradient norm: 1.0 ]8;id=122104;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\189]8;;\
11
-   INFO  | >> PyTorch version 2.7.0 available. ]8;id=514883;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py\config.py]8;;\:]8;id=417488;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py#54\54]8;;\
12
- Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 170085.32it/s]
13
- Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 186828.69it/s]
14
- [15:03:17 04/07] INFO  | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\192]8;;\
15
-   INFO  | >> |=> Training dataset size: 45,530 ]8;id=603472;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\193]8;;\
16
-   INFO  | >> |=> Val dataset size: 45,530 ]8;id=548632;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\195]8;;\
17
-   INFO  | >> [*] Initialize optimizers and schedulers... ]8;id=492873;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\198]8;;\
18
-   INFO  | >> [*] ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\202]8;;\
19
-   INFO  | >> |=> Num training examples = 45530 ]8;id=974614;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\203]8;;\
20
-   INFO  | >> |=> Max training Epochs = 113 ]8;id=574775;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\204]8;;\
21
-   INFO  | >> |=> Total optimization steps = 40000 ]8;id=927413;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\205]8;;\
22
-   INFO  | >> |=> Num steps Per Epoch = 356 ]8;id=9727;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\206]8;;\
23
-   INFO  | >> |=> Effective training epochs = 449.44 ]8;id=842580;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\207]8;;\
24
-   INFO  | >> |=> Global train batch size (w. parallel, distributed & accumulation) = 128 ]8;id=272745;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\208]8;;\
25
-   INFO  | >> |=> Device train batch size = 32 ]8;id=521307;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\209]8;;\
26
-   INFO  | >> |=> Gradient Accumulation steps = 1 ]8;id=871789;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\210]8;;\
27
-   INFO  | >> |=> Num processes (GPUs) = 4 ]8;id=459749;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\211]8;;\
28
- [15:03:19 04/07] INFO  | >> [*] Accelerator runs in: .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503 ]8;id=966756;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\218]8;;\
29
- Traing steps: 25%|██████████████████████▏ | 9999/40000 [46:38<1:13:12, 6.83it/s, loss=0.0668, lr=8.7e-05]
30
- [2026-04-07 15:03:24,386] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
31
- [15:03:24 04/07] INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=680052;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=279409;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
32
-   /tmp/tmpmobd8a5b/test.c -o /tmp/tmpmobd8a5b/test.o  
33
-   INFO  | >> cc -pthread /tmp/tmpmobd8a5b/test.o -laio -o /tmp/tmpmobd8a5b/a.out ]8;id=204038;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=783864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
34
- [15:03:25 04/07] INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=960999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=14714;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
35
-   /tmp/tmp_8xrvwrl/test.c -o /tmp/tmp_8xrvwrl/test.o  
36
-   INFO  | >> cc -pthread /tmp/tmp_8xrvwrl/test.o -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o /tmp/tmp_8xrvwrl/a.out ]8;id=573141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=977459;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
37
-   INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=208448;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=259876;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
38
-   /tmp/tmpawj10h7g/test.c -o /tmp/tmpawj10h7g/test.o  
39
-   INFO  | >> cc -pthread /tmp/tmpawj10h7g/test.o -laio -o /tmp/tmpawj10h7g/a.out ]8;id=572992;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=281580;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
40
- [2026-04-07 15:03:25,792] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
41
-
42
- [15:27:09 04/07] INFO  | >> Saving current state to ]8;id=328668;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=696864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
43
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000  
44
- [15:27:11 04/07] INFO  | >> Model weights saved in ]8;id=515887;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=723262;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
45
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000/model.safetensors  
46
-   INFO  | >> Optimizer state saved in ]8;id=495959;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=616494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
47
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000/optimizer.bin  
48
-   INFO  | >> Scheduler state saved in ]8;id=304800;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=88534;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
49
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000/scheduler.bin  
50
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=821494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=820417;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
51
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000/sampler.bin  
52
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=432054;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=409371;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
53
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000/sampler_1.bin  
54
-   INFO  | >> Random states saved in ]8;id=394271;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=14818;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
55
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000/random_states_0.pkl  
56
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_5000
57
- [15:49:59 04/07] INFO  | >> Saving current state to ]8;id=53870;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=874302;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
58
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000  
59
- [15:50:00 04/07] INFO  | >> Model weights saved in ]8;id=806914;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=990978;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
60
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000/model.safetensors  
61
-   INFO  | >> Optimizer state saved in ]8;id=343681;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=715961;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
62
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000/optimizer.bin  
63
-   INFO  | >> Scheduler state saved in ]8;id=127268;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=521739;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
64
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000/scheduler.bin  
65
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=844039;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=179027;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
66
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000/sampler.bin  
67
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=454670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=653622;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
68
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000/sampler_1.bin  
69
-   INFO  | >> Random states saved in ]8;id=711457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=745414;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
70
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000/random_states_0.pkl  
71
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_10000
72
- [16:12:47 04/07] INFO  | >> Saving current state to ]8;id=204990;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=678670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
73
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000  
74
- [16:12:48 04/07] INFO  | >> Model weights saved in ]8;id=571229;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=976178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
75
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000/model.safetensors  
76
- [16:12:49 04/07] INFO  | >> Optimizer state saved in ]8;id=791370;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=671982;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
77
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000/optimizer.bin  
78
-   INFO  | >> Scheduler state saved in ]8;id=920457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=21464;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
79
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000/scheduler.bin  
80
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=823560;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=925542;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
81
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000/sampler.bin  
82
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=693342;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=885391;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
83
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000/sampler_1.bin  
84
-   INFO  | >> Random states saved in ]8;id=558902;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=434201;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
85
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000/random_states_0.pkl  
86
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_15000
87
- [16:35:30 04/07] INFO  | >> Saving current state to ]8;id=196292;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=360779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
88
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000  
89
- [16:35:31 04/07] INFO  | >> Model weights saved in ]8;id=818149;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=854583;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
90
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000/model.safetensors  
91
-   INFO  | >> Optimizer state saved in ]8;id=341614;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=544562;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
92
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000/optimizer.bin  
93
-   INFO  | >> Scheduler state saved in ]8;id=835298;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=550186;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
94
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000/scheduler.bin  
95
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=889056;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=224742;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
96
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000/sampler.bin  
97
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=921300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=476172;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
98
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000/sampler_1.bin  
99
-   INFO  | >> Random states saved in ]8;id=821364;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=874045;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
100
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000/random_states_0.pkl  
101
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_20000
102
- [16:58:18 04/07] INFO  | >> Saving current state to ]8;id=28505;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=369046;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
103
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000  
104
- [16:58:19 04/07] INFO  | >> Model weights saved in ]8;id=720536;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=600133;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
105
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000/model.safetensors  
106
-   INFO  | >> Optimizer state saved in ]8;id=650939;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=750793;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
107
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000/optimizer.bin  
108
-   INFO  | >> Scheduler state saved in ]8;id=766478;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=224851;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
109
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000/scheduler.bin  
110
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=702868;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=582729;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
111
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000/sampler.bin  
112
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=101114;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=457234;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
113
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000/sampler_1.bin  
114
-   INFO  | >> Random states saved in ]8;id=795987;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=373558;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
115
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000/random_states_0.pkl  
116
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_25000
117
- [17:21:02 04/07] INFO  | >> Saving current state to ]8;id=660693;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=111484;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
118
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000  
119
- [17:21:03 04/07] INFO  | >> Model weights saved in ]8;id=878068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=67019;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
120
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000/model.safetensors  
121
- [17:21:04 04/07] INFO  | >> Optimizer state saved in ]8;id=273529;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=268565;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
122
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000/optimizer.bin  
123
-   INFO  | >> Scheduler state saved in ]8;id=941677;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=948866;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
124
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000/scheduler.bin  
125
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=60095;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=101135;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
126
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000/sampler.bin  
127
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=98429;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=719141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
128
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000/sampler_1.bin  
129
-   INFO  | >> Random states saved in ]8;id=367323;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=330274;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
130
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000/random_states_0.pkl  
131
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_30000
132
- [17:43:46 04/07] INFO  | >> Saving current state to ]8;id=686520;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=748910;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
133
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000  
134
- [17:43:47 04/07] INFO  | >> Model weights saved in ]8;id=331416;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=637309;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
135
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000/model.safetensors  
136
- [17:43:48 04/07] INFO  | >> Optimizer state saved in ]8;id=708752;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=921072;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
137
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000/optimizer.bin  
138
-   INFO  | >> Scheduler state saved in ]8;id=525611;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=714503;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
139
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000/scheduler.bin  
140
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=993068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=856493;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
141
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000/sampler.bin  
142
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=457956;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=921087;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
143
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000/sampler_1.bin  
144
-   INFO  | >> Random states saved in ]8;id=303621;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=762447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
145
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000/random_states_0.pkl  
146
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_35000
147
- [18:06:36 04/07] INFO  | >> Saving current state to ]8;id=311366;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=805194;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
148
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000  
149
-   INFO  | >> Model weights saved in ]8;id=21254;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=568492;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
150
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/model.safetensors  
151
- [18:06:37 04/07] INFO  | >> Optimizer state saved in ]8;id=756734;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=118067;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
152
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/optimizer.bin  
153
-   INFO  | >> Scheduler state saved in ]8;id=39131;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=131787;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
154
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/scheduler.bin  
155
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=687655;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=316922;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
156
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/sampler.bin  
157
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=373263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=435471;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
158
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/sampler_1.bin  
159
-   INFO  | >> Random states saved in ]8;id=441435;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=694779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
160
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000/random_states_0.pkl  
161
- Saved state to .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_40000
162
- Training has reached maximum steps.
163
- [18:06:43 04/07] INFO  | >> Saving current state to ]8;id=569300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=29797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
164
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_39999  
165
- [18:06:44 04/07] INFO  | >> Model weights saved in ]8;id=716966;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=581852;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
166
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_39999/model.safetensors  
167
-   INFO  | >> Optimizer state saved in ]8;id=154857;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=239343;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
168
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_39999/optimizer.bin  
169
-   INFO  | >> Scheduler state saved in ]8;id=502103;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=438178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
170
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_39999/scheduler.bin  
171
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=657224;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=622487;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
172
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_39999/sampler.bin  
173
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=39855;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=681797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
174
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_39999/sampler_1.bin  
175
-   INFO  | >> Random states saved in ]8;id=278106;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=357263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
176
-   .runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/checkpoints/ckpt_39999/random_states_0.pkl  
177
-   INFO  | >> [*] Finalized ACT Trainer. Epoch losses: [0.0, 2.2404654026031494, 1.3566347360610962, 0.6565650701522827, 0.3426889479160309, ]8;id=58988;file:///data/jliu/psi/src/psi/trainers/act_g1.py\act_g1.py]8;;\:]8;id=689824;file:///data/jliu/psi/src/psi/trainers/act_g1.py#331\331]8;;\
178
-   0.2243376225233078]  
179
-   INFO  | >> [*] Happy Ending! ]8;id=61128;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=232614;file:///data/jliu/psi/scripts/train.py#310\310]8;;\
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/requirements.txt DELETED
@@ -1,219 +0,0 @@
1
- tifffile==2025.5.10
2
- mpmath==1.3.0
3
- pynput==1.8.1
4
- exceptiongroup==1.3.0
5
- sympy==1.14.0
6
- decorator==5.2.1
7
- aiohttp-cors==0.8.1
8
- soupsieve==2.8
9
- Farama-Notifications==0.0.4
10
- numpydantic==1.6.7
11
- uvicorn==0.38.0
12
- waterbear==2.6.8
13
- comm==0.2.3
14
- cmake==4.2.3
15
- xxhash==3.5.0
16
- parso==0.8.6
17
- nvidia-cusparse-cu12==12.5.4.2
18
- annotated-doc==0.0.4
19
- jsonlines==4.0.0
20
- nvidia-cuda-runtime-cu12==12.6.77
21
- pydantic==2.10.6
22
- multidict==6.6.4
23
- shtab==1.7.2
24
- nvidia-cuda-nvrtc-cu12==12.6.77
25
- matplotlib-inline==0.2.1
26
- pycollada==0.9.2
27
- pure_eval==0.2.3
28
- beautifulsoup4==4.14.2
29
- msgspec==0.19.0
30
- huggingface-hub==0.35.3
31
- pydantic_core==2.27.2
32
- pytz==2025.2
33
- gymnasium==1.2.3
34
- attrs==25.3.0
35
- requests==2.32.5
36
- hf_transfer==0.1.9
37
- einx==0.3.0
38
- scipy==1.15.3
39
- transforms3d==0.4.2
40
- pyzmq==27.1.0
41
- aiohappyeyeballs==2.6.1
42
- six==1.17.0
43
- expandvars==1.1.2
44
- prompt_toolkit==3.0.52
45
- wcwidth==0.6.0
46
- stack-data==0.6.3
47
- ruamel.yaml.clib==0.2.14
48
- multiprocess==0.70.16
49
- vhacdx==0.0.9
50
- tokenizers==0.22.2
51
- aiosignal==1.4.0
52
- itsdangerous==2.2.0
53
- torchvision==0.22.0
54
- nvidia-cublas-cu12==12.6.4.1
55
- httpx==0.28.1
56
- platformdirs==4.4.0
57
- sentry-sdk==2.39.0
58
- plotly==6.2.0
59
- nvidia-cudnn-cu12==9.5.1.17
60
- jsonschema==4.25.1
61
- fastapi==0.119.1
62
- fsspec==2025.3.0
63
- MarkupSafe==3.0.3
64
- pydantic-yaml==1.6.0
65
- h11==0.16.0
66
- typer==0.24.1
67
- mypy_extensions==1.1.0
68
- nest-asyncio==1.6.0
69
- h5py==3.14.0
70
- python-xlib==0.33
71
- lazy_loader==0.4
72
- einops==0.8.1
73
- albumentations==1.4.18
74
- params_proto==2.13.2
75
- psutil==7.1.0
76
- starlette==0.48.0
77
- anyio==4.11.0
78
- gdown==5.2.0
79
- charset-normalizer==3.4.3
80
- tyro==0.9.32
81
- filelock==3.19.1
82
- websockets==15.0.1
83
- nvidia-cuda-cupti-cu12==12.6.80
84
- python-dotenv==1.2.1
85
- orderly-set==5.5.0
86
- inquirerpy==0.3.4
87
- pexpect==4.9.0
88
- ipython==8.38.0
89
- urllib3==2.5.0
90
- diffusers==0.37.0
91
- PyYAML==6.0.3
92
- antlr4-python3-runtime==4.9.3
93
- mdurl==0.1.2
94
- omegaconf==2.3.0
95
- rerun-sdk==0.22.1
96
- draccus==0.10.0
97
- sentencepiece==0.2.1
98
- referencing==0.37.0
99
- docstring_parser==0.17.0
100
- protobuf==6.33.5
101
- wandb==0.25.1
102
- numpy==1.26.4
103
- GitPython==3.1.45
104
- opencv-python-headless==4.11.0.86
105
- yourdfpy==0.0.58
106
- async-timeout==4.0.3
107
- shapely==2.1.2
108
- frozenlist==1.7.0
109
- simplejpeg==1.9.0
110
- asttokens==3.0.1
111
- Pygments==2.19.2
112
- py-cpuinfo==9.0.0
113
- rtree==1.4.1
114
- jedi==0.19.2
115
- gitdb==4.0.12
116
- cloudpickle==3.1.1
117
- deepspeed==0.17.1
118
- executing==2.2.1
119
- nvidia-nvjitlink-cu12==12.6.85
120
- colorlog==6.10.1
121
- qwen-vl-utils==0.0.14
122
- Werkzeug==3.1.6
123
- zipp==3.23.0
124
- setuptools==80.9.0
125
- albucore==0.0.17
126
- transformers==4.57.0
127
- imageio==2.34.2
128
- frozendict==2.4.6
129
- hjson==3.1.0
130
- jsonschema-specifications==2025.9.1
131
- rpds-py==0.28.0
132
- tornado==6.5.5
133
- blinker==1.9.0
134
- accelerate==1.7.0
135
- argparse==1.4.0
136
- msgpack==1.1.1
137
- smmap==5.0.2
138
- nvidia-nvtx-cu12==12.6.77
139
- packaging==25.0
140
- embreex==2.17.7.post7
141
- vuer==0.0.68
142
- propcache==0.3.2
143
- nvidia-cusolver-cu12==11.7.1.2
144
- triton==3.3.0
145
- pfzy==0.3.4
146
- debugpy==1.8.20
147
- torchcodec==0.4.0
148
- dm-tree==0.1.8
149
- ninja==1.13.0
150
- ipykernel==7.2.0
151
- traitlets==5.14.3
152
- nvidia-cufile-cu12==1.11.1.6
153
- datasets==3.6.0
154
- nvidia-curand-cu12==10.3.7.77
155
- flash_attn==2.7.4.post1
156
- av==16.0.1
157
- sniffio==1.3.1
158
- Jinja2==3.1.6
159
- narwhals==2.6.0
160
- yarl==1.20.1
161
- typeguard==4.4.4
162
- termcolor==3.1.0
163
- jupyter_client==8.8.0
164
- importlib_metadata==8.7.1
165
- eval_type_backport==0.2.2
166
- certifi==2025.8.3
167
- tzdata==2025.2
168
- typing_extensions==4.15.0
169
- viser==1.0.15
170
- typing-inspect==0.9.0
171
- annotated-types==0.7.0
172
- svg.path==7.0
173
- mergedeep==1.3.4
174
- ptyprocess==0.7.0
175
- psi==0.0.0
176
- deepdiff==8.6.1
177
- toml==0.10.2
178
- click==8.3.0
179
- dotenv==0.9.9
180
- trimesh==4.8.3
181
- nvidia-nccl-cu12==2.26.2
182
- pyyaml-include==1.4.1
183
- scikit-image==0.25.2
184
- PySocks==1.7.1
185
- aiohttp==3.10.5
186
- opencv-python==4.11.0.86
187
- nodeenv==1.9.1
188
- lerobot==0.3.3
189
- pillow==11.3.0
190
- idna==3.10
191
- Flask==3.1.3
192
- killport==1.2.0
193
- hf-xet==1.1.10
194
- imageio-ffmpeg==0.6.0
195
- httpcore==1.0.9
196
- nvidia-cusparselt-cu12==0.6.3
197
- networkx==3.4.2
198
- ruamel.yaml==0.18.15
199
- manifold3d==3.2.1
200
- shellingham==1.5.4
201
- nvidia-cufft-cu12==11.3.0.4
202
- mapbox_earcut==1.0.3
203
- peft==0.17.1
204
- pyarrow==21.0.0
205
- pyserial==3.5
206
- torch==2.7.0
207
- dill==0.3.8
208
- markdown-it-py==4.0.0
209
- tqdm==4.67.1
210
- argcomplete==3.6.2
211
- jupyter_core==5.9.1
212
- pandas==2.3.3
213
- lxml==6.0.2
214
- safetensors==0.6.2
215
- python-dateutil==2.9.0.post0
216
- regex==2025.9.18
217
- evdev==1.9.3
218
- rich==14.1.0
219
- psi==0.0.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-metadata.json DELETED
@@ -1,132 +0,0 @@
1
- {
2
- "os": "Linux-6.8.0-106-generic-x86_64-with-glibc2.39",
3
- "python": "CPython 3.10.20",
4
- "startedAt": "2026-04-07T15:03:14.309147Z",
5
- "args": [
6
- "simple_act_config",
7
- "--seed=2026",
8
- "--exp=g1wholebodyhandover-v0",
9
- "--train.name=act-g1",
10
- "--log.report-to=wandb",
11
- "--train.data_parallel=ddp",
12
- "--train.mixed_precision=bf16",
13
- "--train.train-batch-size=32",
14
- "--train.warmup-steps=1000",
15
- "--train.warmup-ratio=None",
16
- "--train.checkpointing-steps=5000",
17
- "--train.validation_steps=500",
18
- "--train.val_num_batches=20",
19
- "--train.gradient_accumulation_steps=1",
20
- "--train.max-training-steps=40000",
21
- "--train.learning-rate=1e-4",
22
- "--train.max-grad-norm=1.0",
23
- "--train.lr_scheduler_kwargs.weight_decay=1e-6",
24
- "--train.lr_scheduler_kwargs.betas",
25
- "0.95",
26
- "0.999",
27
- "--train.lr_scheduler_type=cosine",
28
- "--data.root_dir=/data/jliu/data",
29
- "--data.train-repo-ids=G1WholebodyHandover-v0",
30
- "--data.transform.repack.action_chunk_size=30",
31
- "--data.transform.repack.pad-action-dim=36",
32
- "--data.transform.repack.pad-state-dim=36",
33
- "--data.transform.field.stat-path=meta/stats_psi0.json",
34
- "--data.transform.field.stat-action-key=action",
35
- "--data.transform.field.stat-state-key=states",
36
- "--data.transform.field.normalize-state",
37
- "--data.transform.field.action-norm-type=bounds",
38
- "--data.transform.field.pad-action-dim=36",
39
- "--data.transform.field.pad-state-dim=36",
40
- "--data.transform.model.img-aug",
41
- "--model.chunk-size=30",
42
- "--model.n-action-steps=30",
43
- "--model.action-dim=36",
44
- "--model.state-dim=36",
45
- "--model.use-vae",
46
- "--model.kl-weight=10.0"
47
- ],
48
- "program": "/data/jliu/psi/scripts/train.py",
49
- "codePath": "scripts/train.py",
50
- "codePathLocal": "scripts/train.py",
51
- "git": {
52
- "remote": "https://github.com/songlin/psi.git",
53
- "commit": "04614628ecb677f5f278e2d31c1103aed8127f26"
54
- },
55
- "email": "jliu530@163.com",
56
- "root": "/data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503",
57
- "host": "nebula99",
58
- "executable": "/data/jliu/psi/.venv-psi/bin/python3",
59
- "cpu_count": 128,
60
- "cpu_count_logical": 128,
61
- "gpu": "NVIDIA A100-SXM4-80GB",
62
- "gpu_count": 8,
63
- "disk": {
64
- "/": {
65
- "total": "105089261568",
66
- "used": "89297772544"
67
- }
68
- },
69
- "memory": {
70
- "total": "1623177445376"
71
- },
72
- "gpu_nvidia": [
73
- {
74
- "name": "NVIDIA A100-SXM4-80GB",
75
- "memoryTotal": "85899345920",
76
- "cudaCores": 6912,
77
- "architecture": "Ampere",
78
- "uuid": "GPU-02b51758-2aec-8c71-c751-92567fdb15ad"
79
- },
80
- {
81
- "name": "NVIDIA A100-SXM4-80GB",
82
- "memoryTotal": "85899345920",
83
- "cudaCores": 6912,
84
- "architecture": "Ampere",
85
- "uuid": "GPU-be645bb4-1ef0-15da-fc6b-53c5378650df"
86
- },
87
- {
88
- "name": "NVIDIA A100-SXM4-80GB",
89
- "memoryTotal": "85899345920",
90
- "cudaCores": 6912,
91
- "architecture": "Ampere",
92
- "uuid": "GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1"
93
- },
94
- {
95
- "name": "NVIDIA A100-SXM4-80GB",
96
- "memoryTotal": "85899345920",
97
- "cudaCores": 6912,
98
- "architecture": "Ampere",
99
- "uuid": "GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b"
100
- },
101
- {
102
- "name": "NVIDIA A100-SXM4-80GB",
103
- "memoryTotal": "85899345920",
104
- "cudaCores": 6912,
105
- "architecture": "Ampere",
106
- "uuid": "GPU-57787183-4904-b2c9-cf52-1c1af976b365"
107
- },
108
- {
109
- "name": "NVIDIA A100-SXM4-80GB",
110
- "memoryTotal": "85899345920",
111
- "cudaCores": 6912,
112
- "architecture": "Ampere",
113
- "uuid": "GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3"
114
- },
115
- {
116
- "name": "NVIDIA A100-SXM4-80GB",
117
- "memoryTotal": "85899345920",
118
- "cudaCores": 6912,
119
- "architecture": "Ampere",
120
- "uuid": "GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c"
121
- },
122
- {
123
- "name": "NVIDIA A100-SXM4-80GB",
124
- "memoryTotal": "85899345920",
125
- "cudaCores": 6912,
126
- "architecture": "Ampere",
127
- "uuid": "GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8"
128
- }
129
- ],
130
- "cudaVersion": "12.9",
131
- "writerId": "1ney3xhb33gta4c2m7gfcmat7y4xd0vk"
132
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/files/wandb-summary.json DELETED
@@ -1 +0,0 @@
1
- {"train/epoch":112,"val/denorm_err_l1_height":0,"val/denorm_err_l1_torso_target_yaw":0.00029178752447478473,"val/denorm_err_l1_rpy":0.002766967751085758,"val/denorm_err_l1_torso_vy":0.0009157538297586143,"val/denorm_err_l1_arm_joints":0.020835014060139656,"val/denorm_err_l1_torso_vyaw":0.0029068603180348873,"val/denorm_err_l1_torso_vx":0.0011786402901634574,"val/bc_loss":-0.0009642460499890149,"val/denorm_err_l1_hand_joints":0.0035619293339550495,"_step":40000,"val/kld_loss":-0.0008920701220631599,"_wandb":{"runtime":11009},"train/loss":0.00894236471503973,"_timestamp":1.7755852034701765e+09,"train/kld_loss":1.95428729057312e-05,"train/l1_loss":0.008746935985982418,"_runtime":11009.454112423}
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-core.log DELETED
@@ -1,19 +0,0 @@
1
- {"time":"2026-04-07T15:03:14.405213784Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9da_xq2v/port-2236517.txt","pid":2236517,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
- {"time":"2026-04-07T15:03:14.405914689Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2236517-2462798-4161116888/socket","Net":"unix"}}
3
- {"time":"2026-04-07T15:03:14.40607627Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2236517}
4
- {"time":"2026-04-07T15:03:14.567096386Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
5
- {"time":"2026-04-07T15:03:14.579495643Z","level":"INFO","msg":"handleInformInit: received","streamId":"26s2ml9t","id":"1(@)"}
6
- {"time":"2026-04-07T15:03:14.911097462Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"26s2ml9t","id":"1(@)"}
7
- {"time":"2026-04-07T15:03:20.630060907Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fx3xgud82qm3"}
8
- {"time":"2026-04-07T18:06:44.915316242Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fx3xgud82qm3"}
9
- {"time":"2026-04-07T18:06:45.754451187Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"fx3xgud82qm3"}
10
- {"time":"2026-04-07T18:06:45.755226903Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"26s2ml9t","id":"1(@)"}
11
- {"time":"2026-04-07T18:06:45.755767328Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"26s2ml9t","id":"1(@)"}
12
- {"time":"2026-04-07T18:07:47.115746568Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
13
- {"time":"2026-04-07T18:07:47.115798628Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
14
- {"time":"2026-04-07T18:07:47.115808058Z","level":"INFO","msg":"server is shutting down"}
15
- {"time":"2026-04-07T18:07:47.115888389Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2236517-2462798-4161116888/socket","Net":"unix"}}
16
- {"time":"2026-04-07T18:07:47.115927399Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
17
- {"time":"2026-04-07T18:07:47.11596726Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
18
- {"time":"2026-04-07T18:07:47.11597218Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
19
- {"time":"2026-04-07T18:07:47.11597774Z","level":"INFO","msg":"server is closed"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log DELETED
The diff for this file is too large to render. See raw diff
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log DELETED
@@ -1,25 +0,0 @@
1
- 2026-04-07 15:03:14,310 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
2
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Configure stats pid to 2236517
3
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug.log
5
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodyhandover-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/logs/debug-internal.log
6
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():844] calling init triggers
7
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
- config: {'_wandb': {}}
9
- 2026-04-07 15:03:14,311 INFO MainThread:2236517 [wandb_init.py:init():892] starting backend
10
- 2026-04-07 15:03:14,567 INFO MainThread:2236517 [wandb_init.py:init():895] sending inform_init request
11
- 2026-04-07 15:03:14,577 INFO MainThread:2236517 [wandb_init.py:init():903] backend started and connected
12
- 2026-04-07 15:03:14,578 INFO MainThread:2236517 [wandb_init.py:init():973] updated telemetry
13
- 2026-04-07 15:03:14,584 INFO MainThread:2236517 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
- 2026-04-07 15:03:15,461 INFO MainThread:2236517 [wandb_init.py:init():1042] starting run threads in backend
15
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_console_start():2524] atexit reg
16
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
- 2026-04-07 15:03:15,617 INFO MainThread:2236517 [wandb_run.py:_redirect():2465] Redirects installed.
19
- 2026-04-07 15:03:15,625 INFO MainThread:2236517 [wandb_init.py:init():1082] run started, returning control to user process
20
- 2026-04-07 15:03:15,626 INFO MainThread:2236517 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodyhandover-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604071503', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-0.5, -1.1095792134107943e-16, -1.1095792134107943e-16, -1.5, -1.5, -1.5, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.2166350627321588e-16, -2.2166350627321588e-16, 0.0, -2.2166350627321588e-16, -0.47567468881607056, 0.1900009959936142, -0.512170135974884, -0.6265152096748352, -0.5008617043495178, -0.8220608830451965, -0.9223371148109436, -0.49507391452789307, -0.3437551259994507, -0.6871383190155029, -0.7637607455253601, -0.7568023204803467, -0.576077401638031, -0.4588268995285034, -0.13876836001873016, -0.10360867530107498, -0.47856518626213074, 0.7400000095367432, -0.5, -0.5, -0.26161932945251465, -0.06718750298023224], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 2.2146225653890418e-16, 2.2146225653890418e-16, 2.2146225653890418e-16, 1.2266071310501902e-19, 1.1078670818917075e-16, 1.1078670818917075e-16, 1.5, 1.5, 0.6000000238418579, 1.5, 0.2472410947084427, 0.7092280983924866, 1.2571598291397095, 0.42311304807662964, 0.8564174771308899, 0.5002086162567139, 0.5172277092933655, 0.16140148043632507, -0.1900009959936142, 0.5362864136695862, 0.5715147256851196, 0.5002322196960449, 0.566592276096344, 0.6392397880554199, 0.1580466777086258, 0.2233395129442215, 0.2582152187824249, 0.7400000095367432, 0.5, 0.5, 0.3454970121383667, 0.2899305522441864], 'state_min': [-0.5564982891082764, -0.48307520151138306, -0.0005447770818136632, -0.8388738632202148, -1.3970016241073608, -0.8296014666557312, -1.4599460363388062, -0.5806806683540344, -0.5149835348129272, -0.6775947213172913, -0.001480442238971591, -0.0002713006397243589, -0.000914653530344367, -0.00019419840828049928, -0.4206617772579193, 0.13972464203834534, -0.546251654624939, -0.5596316456794739, -0.4764360189437866, -0.7253566384315491, -0.9443663954734802, -0.4381798803806305, -0.3338131606578827, -0.667724072933197, -0.6881827116012573, -0.7544379830360413, -0.5189417600631714, -0.4484957158565521, -0.13709338009357452, -0.07360810041427612, -0.4748336970806122, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.43566983938217163, 0.3739710748195648, 0.6575677990913391, 0.004060761071741581, 0.0005700877518393099, 0.0004725759499706328, 0.00010080631182063371, 1.310737025050912e-05, 0.21882089972496033, 0.0005271440604701638, 0.530737042427063, 1.4406861066818237, 1.4605127573013306, 1.4595911502838135, 0.2663630545139313, 0.657910943031311, 1.2515853643417358, 0.502498209476471, 0.8292973637580872, 0.5248894095420837, 0.4653257131576538, 0.18638382852077484, -0.16696421802043915, 0.49318820238113403, 0.6363148093223572, 0.45773962140083313, 0.6238265037536621, 0.653800904750824, 0.1436084657907486, 0.25937986373901367, 0.26422709226608276, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyHandover-v0'], 'val_repo_ids': ['G1WholebodyHandover-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '40557'}}
21
- 2026-04-07 18:06:44,913 INFO MainThread:2236517 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/26s2ml9t
22
- 2026-04-07 18:06:44,914 INFO MainThread:2236517 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
23
- 2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2472] restore
24
- 2026-04-07 18:06:44,915 INFO MainThread:2236517 [wandb_run.py:_restore():2478] restore done
25
- 2026-04-07 18:06:45,754 INFO MainThread:2236517 [wandb_run.py:_footer_sync_info():3868] logging synced files
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodyhandoverteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604071503/wandb/run-20260407_150314-26s2ml9t/run-26s2ml9t.wandb DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9677d373b978274325f092362db630268d9300af1974e711763c57ac318d130e
3
- size 47853675
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/argv.txt DELETED
@@ -1,40 +0,0 @@
1
- scripts/train.py
2
- simple_act_config
3
- --seed=2026
4
- --exp=g1wholebodylocomotionpickbetweentablesteleop-v0
5
- --train.name=act-g1
6
- --log.report-to=wandb
7
- --train.data_parallel=ddp
8
- --train.mixed_precision=bf16
9
- --train.train-batch-size=32
10
- --train.warmup-steps=1000
11
- --train.warmup-ratio=None
12
- --train.checkpointing-steps=5000
13
- --train.validation_steps=500
14
- --train.val_num_batches=20
15
- --train.gradient_accumulation_steps=1
16
- --train.max-training-steps=40000
17
- --train.learning-rate=1e-4
18
- --train.max-grad-norm=1.0
19
- --train.lr_scheduler_kwargs.weight_decay=1e-6
20
- --train.lr_scheduler_kwargs.betas 0.95 0.999
21
- --train.lr_scheduler_type=cosine
22
- --data.root_dir=/data/jliu/data
23
- --data.train-repo-ids=G1WholebodyLocomotionPickBetweenTablesTeleop-v0
24
- --data.transform.repack.action_chunk_size=30
25
- --data.transform.repack.pad-action-dim=36
26
- --data.transform.repack.pad-state-dim=36
27
- --data.transform.field.stat-path=meta/stats_psi0.json
28
- --data.transform.field.stat-action-key=action
29
- --data.transform.field.stat-state-key=states
30
- --data.transform.field.normalize-state
31
- --data.transform.field.action-norm-type=bounds
32
- --data.transform.field.pad-action-dim=36
33
- --data.transform.field.pad-state-dim=36
34
- --data.transform.model.img-aug
35
- --model.chunk-size=30
36
- --model.n-action-steps=30
37
- --model.action-dim=36
38
- --model.state-dim=36
39
- --model.use-vae
40
- --model.kl-weight=10.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4535d4f0decb1422ba53c4f3d3d639f9362638e31a1cf4bf34a2cccb860c6d57
3
- size 206658952
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/optimizer.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:71a3663f16b5fc24065782c149cc1257801c83b62702f58b729a1fc345c037f0
3
- size 413111371
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/random_states_0.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9d45c9d781594007d7b0976207a38ab9be25b60fe7bd7c88e747c0ad60723cea
3
- size 15473
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/scheduler.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:edd2ef4de5774723b53b76baced1d134ab2a534e3951c7a3ebfbc157468c8f72
3
- size 1401
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/envs.txt DELETED
@@ -1,17 +0,0 @@
1
- OMP_NUM_THREADS=32
2
- HF_HOME=/data/cache
3
- TORCH_HOME=/data/cache
4
- HF_TOKEN=hf_...TiKa
5
- HF_LEROBOT_HOME=/data/data/lerobot
6
- WE_HOME=Not Set
7
- DATA_HOME=/data/data
8
- UV_CACHE_DIR=/data/cache
9
- WANDB_API_KEY=90e...5c06
10
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
11
- CUDA_VISIBLE_DEVICES=4,5,6,7
12
- WORLD_SIZE=4
13
- LOCAL_WORLD_SIZE=4
14
- RANK=0
15
- LOCAL_RANK=0
16
- MASTER_ADDR=nebula99
17
- MASTER_PORT=35899
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/run_config.json DELETED
@@ -1,310 +0,0 @@
1
- {
2
- "exp": "g1wholebodylocomotionpickbetweentablesteleop-v0",
3
- "seed": 2026,
4
- "auto_tag_run": false,
5
- "eval": false,
6
- "debug": false,
7
- "timestamp": "2604081550",
8
- "log": {
9
- "logging_dir": "logs",
10
- "report_to": "wandb",
11
- "log_freq": 100
12
- },
13
- "wandb": {
14
- "project": "psi",
15
- "entity": "jliu530-soochow-university",
16
- "group": "act-g1",
17
- "id": "ii0lixdx",
18
- "name": "g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550",
19
- "resume": "allow"
20
- },
21
- "train": {
22
- "num_workers": 8,
23
- "overfit_single_batch": false,
24
- "name": "act-g1",
25
- "resume_from_checkpoint": null,
26
- "skip_resumed_steps": false,
27
- "hf_token": ".hf_token",
28
- "lora": false,
29
- "output_dir": ".runs",
30
- "gradient_accumulation_steps": 1,
31
- "mixed_precision": "bf16",
32
- "max_grad_norm": 1.0,
33
- "optimizer_foreach": null,
34
- "train_batch_size": 32,
35
- "val_batch_size": 16,
36
- "val_num_batches": 20,
37
- "checkpointing_steps": 5000,
38
- "max_checkpoints_to_keep": null,
39
- "validation_steps": 500,
40
- "learning_rate": 0.0001,
41
- "lr_scheduler_type": "cosine",
42
- "lr_scheduler_kwargs": {
43
- "betas": [
44
- 0.95,
45
- 0.999
46
- ],
47
- "weight_decay": 1e-6,
48
- "eps": 1e-8
49
- },
50
- "scheduler_specific_kwargs": {},
51
- "data_parallel": "ddp",
52
- "sharding_strategy": "full-shard",
53
- "deepspeed_config": "/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json",
54
- "enable_gradient_checkpointing": true,
55
- "enable_mixed_precision_training": true,
56
- "reduce_in_full_precision": true,
57
- "max_training_steps": 40000,
58
- "num_train_epochs": null,
59
- "warmup_steps": 1000,
60
- "warmup_ratio": null
61
- },
62
- "data": {
63
- "transform": {
64
- "repack": {
65
- "dataset_name": "simple",
66
- "num_past_frames": 0,
67
- "action_chunk_size": 30,
68
- "pad_action_dim": 36,
69
- "pad_state_dim": 36
70
- },
71
- "model": {
72
- "resize": {
73
- "size": [
74
- 256,
75
- 480
76
- ]
77
- },
78
- "center_crop": {
79
- "size": [
80
- 224,
81
- 224
82
- ]
83
- },
84
- "color_jitter": {
85
- "brightness": 0.2,
86
- "contrast": [
87
- 0.8,
88
- 1.2
89
- ],
90
- "saturation": [
91
- 0.8,
92
- 1.2
93
- ],
94
- "hue": 0.05
95
- },
96
- "normalize": {
97
- "mean": [
98
- 0.485,
99
- 0.456,
100
- 0.406
101
- ],
102
- "std": [
103
- 0.229,
104
- 0.224,
105
- 0.225
106
- ]
107
- },
108
- "img_aug": true
109
- },
110
- "field": {
111
- "stat_path": "meta/stats_psi0.json",
112
- "action_norm_type": "bounds",
113
- "stat_action_key": "action",
114
- "stat_state_key": "states",
115
- "use_norm_mask": false,
116
- "action_norm_masks": [
117
- true,
118
- true,
119
- true,
120
- true,
121
- true,
122
- true,
123
- false
124
- ],
125
- "action_min": [
126
- 0.0,
127
- 0.0,
128
- 0.0,
129
- -1.5,
130
- -1.5,
131
- -0.6000000238418579,
132
- -1.5,
133
- -0.5,
134
- -0.699999988079071,
135
- -0.699999988079071,
136
- -2.216935230032842e-16,
137
- -2.216935230032842e-16,
138
- -4.0845591349633594e-18,
139
- -2.216935230032842e-16,
140
- -0.4883034825325012,
141
- 0.1900009959936142,
142
- -0.5470856428146362,
143
- -0.34318920969963074,
144
- -0.35952919721603394,
145
- -0.35302427411079407,
146
- -0.4469815790653229,
147
- -0.6371198296546936,
148
- -0.7683824300765991,
149
- -1.0653810501098633,
150
- -0.8479154706001282,
151
- -1.0297260284423828,
152
- -0.42936205863952637,
153
- -0.5147944092750549,
154
- -0.16820405423641205,
155
- -0.045328833162784576,
156
- -0.13282617926597595,
157
- 0.7400000095367432,
158
- -0.5,
159
- -0.5,
160
- -1.0,
161
- -3.138223648071289
162
- ],
163
- "action_max": [
164
- 0.5,
165
- 0.699999988079071,
166
- 0.699999988079071,
167
- 0.0,
168
- 0.0,
169
- 0.0,
170
- 0.0,
171
- 6.930528109384597e-19,
172
- 1.108467615016421e-16,
173
- 1.108467615016421e-16,
174
- 1.5,
175
- 1.5,
176
- 1.0,
177
- 1.5,
178
- 0.2557959258556366,
179
- 0.35884979367256165,
180
- 0.5090755820274353,
181
- 0.19132143259048462,
182
- 0.21249642968177795,
183
- 0.3992660641670227,
184
- 0.4283020794391632,
185
- 0.1457289606332779,
186
- -0.1900009959936142,
187
- 0.6150448322296143,
188
- 0.35468167066574097,
189
- 0.8703295588493347,
190
- 0.7531875371932983,
191
- 0.971237301826477,
192
- 0.13985762000083923,
193
- 0.15686897933483124,
194
- 0.4661160111427307,
195
- 0.7400000095367432,
196
- 0.5,
197
- 0.5,
198
- 1.0,
199
- 3.1414895057678223
200
- ],
201
- "state_min": [
202
- -0.02442001923918724,
203
- -0.0517612099647522,
204
- -0.0006534014828503132,
205
- -0.5095356106758118,
206
- -1.323034405708313,
207
- -1.3221508264541626,
208
- -1.3230019807815552,
209
- -0.5770347714424133,
210
- -0.4338151812553406,
211
- -0.6721642017364502,
212
- -0.0017213862156495452,
213
- -7.534810038123396e-7,
214
- -0.001927333534695208,
215
- -1.075333216249419e-6,
216
- -0.43650975823402405,
217
- 0.15721464157104492,
218
- -0.5489339232444763,
219
- -0.2632291913032532,
220
- -0.3508843183517456,
221
- -0.23784859478473663,
222
- -0.4281824827194214,
223
- -0.5803383588790894,
224
- -0.7118590474128723,
225
- -1.0344431400299072,
226
- -0.7932196259498596,
227
- -1.0205217599868774,
228
- -0.3445618450641632,
229
- -0.5986371040344238,
230
- -0.13537253439426422,
231
- -0.0017330688424408436,
232
- -0.1421850621700287,
233
- 0.7400000095367432,
234
- 0.0,
235
- 0.0,
236
- 0.0,
237
- 0.0
238
- ],
239
- "state_max": [
240
- 0.47981399297714233,
241
- 0.6772664189338684,
242
- 0.6746510863304138,
243
- 0.0010172375477850437,
244
- 0.0007091082516126335,
245
- 0.001881288131698966,
246
- 0.0011398319620639086,
247
- 6.141255539660051e-7,
248
- 0.3043450713157654,
249
- 6.343479981296696e-7,
250
- 0.6933000087738037,
251
- 1.4612544775009155,
252
- 1.4651201963424683,
253
- 1.4609057903289795,
254
- 0.2809508740901947,
255
- 0.34028318524360657,
256
- 0.47627460956573486,
257
- 0.26476219296455383,
258
- 0.20825636386871338,
259
- 0.4566418528556824,
260
- 0.42864030599594116,
261
- 0.1656116098165512,
262
- -0.1549365073442459,
263
- 0.5154499411582947,
264
- 0.4242899715900421,
265
- 0.8548054695129395,
266
- 0.8040095567703247,
267
- 0.9811649322509766,
268
- 0.136736661195755,
269
- 0.195722296833992,
270
- 0.45781663060188293,
271
- 0.7400000095367432,
272
- 0.0,
273
- 0.0,
274
- 0.0,
275
- 0.0
276
- ],
277
- "normalize_state": true,
278
- "pad_action_dim": 36,
279
- "pad_state_dim": 36
280
- }
281
- },
282
- "root_dir": "/data/jliu/data",
283
- "train_repo_ids": [
284
- "G1WholebodyLocomotionPickBetweenTablesTeleop-v0"
285
- ],
286
- "val_repo_ids": [
287
- "G1WholebodyLocomotionPickBetweenTablesTeleop-v0"
288
- ]
289
- },
290
- "model": {
291
- "n_obs_steps": 1,
292
- "chunk_size": 30,
293
- "n_action_steps": 30,
294
- "action_dim": 36,
295
- "state_dim": 36,
296
- "dim_model": 512,
297
- "n_heads": 8,
298
- "dim_feedforward": 3200,
299
- "feedforward_activation": "relu",
300
- "n_encoder_layers": 4,
301
- "n_decoder_layers": 1,
302
- "pre_norm": false,
303
- "dropout": 0.1,
304
- "use_vae": true,
305
- "latent_dim": 32,
306
- "n_vae_encoder_layers": 4,
307
- "kl_weight": 10.0,
308
- "temporal_ensemble_coeff": null
309
- }
310
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug-internal.log DELETED
The diff for this file is too large to render. See raw diff
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/debug.log DELETED
@@ -1,25 +0,0 @@
1
- 2026-04-08 15:50:54,515 INFO MainThread:641049 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
2
- 2026-04-08 15:50:54,515 INFO MainThread:641049 [wandb_setup.py:_flush():81] Configure stats pid to 641049
3
- 2026-04-08 15:50:54,515 INFO MainThread:641049 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
- 2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /data/jliu/psi/.runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/logs/debug.log
5
- 2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /data/jliu/psi/.runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/logs/debug-internal.log
6
- 2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:init():844] calling init triggers
7
- 2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
8
- config: {'_wandb': {}}
9
- 2026-04-08 15:50:54,516 INFO MainThread:641049 [wandb_init.py:init():892] starting backend
10
- 2026-04-08 15:50:54,765 INFO MainThread:641049 [wandb_init.py:init():895] sending inform_init request
11
- 2026-04-08 15:50:54,774 INFO MainThread:641049 [wandb_init.py:init():903] backend started and connected
12
- 2026-04-08 15:50:54,775 INFO MainThread:641049 [wandb_init.py:init():973] updated telemetry
13
- 2026-04-08 15:50:54,780 INFO MainThread:641049 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
14
- 2026-04-08 15:50:55,555 INFO MainThread:641049 [wandb_init.py:init():1042] starting run threads in backend
15
- 2026-04-08 15:50:55,706 INFO MainThread:641049 [wandb_run.py:_console_start():2524] atexit reg
16
- 2026-04-08 15:50:55,707 INFO MainThread:641049 [wandb_run.py:_redirect():2373] redirect: wrap_raw
17
- 2026-04-08 15:50:55,707 INFO MainThread:641049 [wandb_run.py:_redirect():2442] Wrapping output streams.
18
- 2026-04-08 15:50:55,707 INFO MainThread:641049 [wandb_run.py:_redirect():2465] Redirects installed.
19
- 2026-04-08 15:50:55,710 INFO MainThread:641049 [wandb_init.py:init():1082] run started, returning control to user process
20
- 2026-04-08 15:50:55,712 INFO MainThread:641049 [wandb_run.py:_config_callback():1403] config_cb None None {'exp': 'g1wholebodylocomotionpickbetweentablesteleop-v0', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2604081550', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'psi', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'optimizer_foreach': None, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 40000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'simple', 'num_past_frames': 0, 'action_chunk_size': 30, 'pad_action_dim': 36, 'pad_state_dim': 36}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}, 'field': {'stat_path': 'meta/stats_psi0.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [0.0, 0.0, 0.0, -1.5, -1.5, -0.6000000238418579, -1.5, -0.5, -0.699999988079071, -0.699999988079071, -2.216935230032842e-16, -2.216935230032842e-16, -4.0845591349633594e-18, -2.216935230032842e-16, -0.4883034825325012, 0.1900009959936142, -0.5470856428146362, -0.34318920969963074, -0.35952919721603394, -0.35302427411079407, -0.4469815790653229, -0.6371198296546936, -0.7683824300765991, -1.0653810501098633, -0.8479154706001282, -1.0297260284423828, -0.42936205863952637, -0.5147944092750549, -0.16820405423641205, -0.045328833162784576, -0.13282617926597595, 0.7400000095367432, -0.5, -0.5, -1.0, -3.138223648071289], 'action_max': [0.5, 0.699999988079071, 0.699999988079071, 0.0, 0.0, 0.0, 0.0, 6.930528109384597e-19, 1.108467615016421e-16, 1.108467615016421e-16, 1.5, 1.5, 1.0, 1.5, 0.2557959258556366, 0.35884979367256165, 0.5090755820274353, 0.19132143259048462, 0.21249642968177795, 0.3992660641670227, 0.4283020794391632, 0.1457289606332779, -0.1900009959936142, 0.6150448322296143, 0.35468167066574097, 0.8703295588493347, 0.7531875371932983, 0.971237301826477, 0.13985762000083923, 0.15686897933483124, 0.4661160111427307, 0.7400000095367432, 0.5, 0.5, 1.0, 3.1414895057678223], 'state_min': [-0.02442001923918724, -0.0517612099647522, -0.0006534014828503132, -0.5095356106758118, -1.323034405708313, -1.3221508264541626, -1.3230019807815552, -0.5770347714424133, -0.4338151812553406, -0.6721642017364502, -0.0017213862156495452, -7.534810038123396e-07, -0.001927333534695208, -1.075333216249419e-06, -0.43650975823402405, 0.15721464157104492, -0.5489339232444763, -0.2632291913032532, -0.3508843183517456, -0.23784859478473663, -0.4281824827194214, -0.5803383588790894, -0.7118590474128723, -1.0344431400299072, -0.7932196259498596, -1.0205217599868774, -0.3445618450641632, -0.5986371040344238, -0.13537253439426422, -0.0017330688424408436, -0.1421850621700287, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'state_max': [0.47981399297714233, 0.6772664189338684, 0.6746510863304138, 0.0010172375477850437, 0.0007091082516126335, 0.001881288131698966, 0.0011398319620639086, 6.141255539660051e-07, 0.3043450713157654, 6.343479981296696e-07, 0.6933000087738037, 1.4612544775009155, 1.4651201963424683, 1.4609057903289795, 0.2809508740901947, 0.34028318524360657, 0.47627460956573486, 0.26476219296455383, 0.20825636386871338, 0.4566418528556824, 0.42864030599594116, 0.1656116098165512, -0.1549365073442459, 0.5154499411582947, 0.4242899715900421, 0.8548054695129395, 0.8040095567703247, 0.9811649322509766, 0.136736661195755, 0.195722296833992, 0.45781663060188293, 0.7400000095367432, 0.0, 0.0, 0.0, 0.0], 'normalize_state': True, 'pad_action_dim': 36, 'pad_state_dim': 36}}, 'root_dir': '/data/jliu/data', 'train_repo_ids': ['G1WholebodyLocomotionPickBetweenTablesTeleop-v0'], 'val_repo_ids': ['G1WholebodyLocomotionPickBetweenTablesTeleop-v0']}, 'model': {'n_obs_steps': 1, 'chunk_size': 30, 'n_action_steps': 30, 'action_dim': 36, 'state_dim': 36, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/data/cache', 'TORCH_HOME': '/data/cache', 'HF_TOKEN': 'hf_...TiKa', 'HF_LEROBOT_HOME': '/data/data/lerobot', 'WE_HOME': 'Not Set', 'DATA_HOME': '/data/data', 'UV_CACHE_DIR': '/data/cache', 'WANDB_API_KEY': '90e...5c06', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '4,5,6,7', 'WORLD_SIZE': '4', 'LOCAL_WORLD_SIZE': '4', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '35899'}}
21
- 2026-04-08 19:34:32,424 INFO MainThread:641049 [wandb_run.py:_finish():2291] finishing run jliu530-soochow-university/psi/ii0lixdx
22
- 2026-04-08 19:34:32,427 INFO MainThread:641049 [wandb_run.py:_atexit_cleanup():2490] got exitcode: 0
23
- 2026-04-08 19:34:32,427 INFO MainThread:641049 [wandb_run.py:_restore():2472] restore
24
- 2026-04-08 19:34:32,427 INFO MainThread:641049 [wandb_run.py:_restore():2478] restore done
25
- 2026-04-08 19:34:33,334 INFO MainThread:641049 [wandb_run.py:_footer_sync_info():3868] logging synced files
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/config.yaml DELETED
@@ -1,448 +0,0 @@
1
- _wandb:
2
- value:
3
- cli_version: 0.25.1
4
- e:
5
- ufhcsemgf4luglllwa36q76bqs2rc5qh:
6
- args:
7
- - simple_act_config
8
- - --seed=2026
9
- - --exp=g1wholebodylocomotionpickbetweentablesteleop-v0
10
- - --train.name=act-g1
11
- - --log.report-to=wandb
12
- - --train.data_parallel=ddp
13
- - --train.mixed_precision=bf16
14
- - --train.train-batch-size=32
15
- - --train.warmup-steps=1000
16
- - --train.warmup-ratio=None
17
- - --train.checkpointing-steps=5000
18
- - --train.validation_steps=500
19
- - --train.val_num_batches=20
20
- - --train.gradient_accumulation_steps=1
21
- - --train.max-training-steps=40000
22
- - --train.learning-rate=1e-4
23
- - --train.max-grad-norm=1.0
24
- - --train.lr_scheduler_kwargs.weight_decay=1e-6
25
- - --train.lr_scheduler_kwargs.betas
26
- - "0.95"
27
- - "0.999"
28
- - --train.lr_scheduler_type=cosine
29
- - --data.root_dir=/data/jliu/data
30
- - --data.train-repo-ids=G1WholebodyLocomotionPickBetweenTablesTeleop-v0
31
- - --data.transform.repack.action_chunk_size=30
32
- - --data.transform.repack.pad-action-dim=36
33
- - --data.transform.repack.pad-state-dim=36
34
- - --data.transform.field.stat-path=meta/stats_psi0.json
35
- - --data.transform.field.stat-action-key=action
36
- - --data.transform.field.stat-state-key=states
37
- - --data.transform.field.normalize-state
38
- - --data.transform.field.action-norm-type=bounds
39
- - --data.transform.field.pad-action-dim=36
40
- - --data.transform.field.pad-state-dim=36
41
- - --data.transform.model.img-aug
42
- - --model.chunk-size=30
43
- - --model.n-action-steps=30
44
- - --model.action-dim=36
45
- - --model.state-dim=36
46
- - --model.use-vae
47
- - --model.kl-weight=10.0
48
- codePath: scripts/train.py
49
- codePathLocal: scripts/train.py
50
- cpu_count: 128
51
- cpu_count_logical: 128
52
- cudaVersion: "12.9"
53
- disk:
54
- /:
55
- total: "105089261568"
56
- used: "92984258560"
57
- email: jliu530@163.com
58
- executable: /data/jliu/psi/.venv-psi/bin/python3
59
- git:
60
- commit: 04614628ecb677f5f278e2d31c1103aed8127f26
61
- remote: https://github.com/songlin/psi.git
62
- gpu: NVIDIA A100-SXM4-80GB
63
- gpu_count: 8
64
- gpu_nvidia:
65
- - architecture: Ampere
66
- cudaCores: 6912
67
- memoryTotal: "85899345920"
68
- name: NVIDIA A100-SXM4-80GB
69
- uuid: GPU-02b51758-2aec-8c71-c751-92567fdb15ad
70
- - architecture: Ampere
71
- cudaCores: 6912
72
- memoryTotal: "85899345920"
73
- name: NVIDIA A100-SXM4-80GB
74
- uuid: GPU-be645bb4-1ef0-15da-fc6b-53c5378650df
75
- - architecture: Ampere
76
- cudaCores: 6912
77
- memoryTotal: "85899345920"
78
- name: NVIDIA A100-SXM4-80GB
79
- uuid: GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1
80
- - architecture: Ampere
81
- cudaCores: 6912
82
- memoryTotal: "85899345920"
83
- name: NVIDIA A100-SXM4-80GB
84
- uuid: GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b
85
- - architecture: Ampere
86
- cudaCores: 6912
87
- memoryTotal: "85899345920"
88
- name: NVIDIA A100-SXM4-80GB
89
- uuid: GPU-57787183-4904-b2c9-cf52-1c1af976b365
90
- - architecture: Ampere
91
- cudaCores: 6912
92
- memoryTotal: "85899345920"
93
- name: NVIDIA A100-SXM4-80GB
94
- uuid: GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3
95
- - architecture: Ampere
96
- cudaCores: 6912
97
- memoryTotal: "85899345920"
98
- name: NVIDIA A100-SXM4-80GB
99
- uuid: GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c
100
- - architecture: Ampere
101
- cudaCores: 6912
102
- memoryTotal: "85899345920"
103
- name: NVIDIA A100-SXM4-80GB
104
- uuid: GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8
105
- host: nebula99
106
- memory:
107
- total: "1623177445376"
108
- os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
109
- program: /data/jliu/psi/scripts/train.py
110
- python: CPython 3.10.20
111
- root: /data/jliu/psi/.runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550
112
- startedAt: "2026-04-08T15:50:54.514093Z"
113
- writerId: ufhcsemgf4luglllwa36q76bqs2rc5qh
114
- m: []
115
- python_version: 3.10.20
116
- t:
117
- "1":
118
- - 1
119
- - 11
120
- - 41
121
- - 49
122
- - 71
123
- "2":
124
- - 1
125
- - 11
126
- - 41
127
- - 49
128
- - 51
129
- - 71
130
- - 83
131
- "3":
132
- - 2
133
- - 13
134
- - 61
135
- "4": 3.10.20
136
- "5": 0.25.1
137
- "6": 4.57.0
138
- "12": 0.25.1
139
- "13": linux-x86_64
140
- auto_tag_run:
141
- value: false
142
- data:
143
- value:
144
- root_dir: /data/jliu/data
145
- train_repo_ids:
146
- - G1WholebodyLocomotionPickBetweenTablesTeleop-v0
147
- transform:
148
- field:
149
- action_max:
150
- - 0.5
151
- - 0.699999988079071
152
- - 0.699999988079071
153
- - 0
154
- - 0
155
- - 0
156
- - 0
157
- - 6.930528109384597e-19
158
- - 1.108467615016421e-16
159
- - 1.108467615016421e-16
160
- - 1.5
161
- - 1.5
162
- - 1
163
- - 1.5
164
- - 0.2557959258556366
165
- - 0.35884979367256165
166
- - 0.5090755820274353
167
- - 0.19132143259048462
168
- - 0.21249642968177795
169
- - 0.3992660641670227
170
- - 0.4283020794391632
171
- - 0.1457289606332779
172
- - -0.1900009959936142
173
- - 0.6150448322296143
174
- - 0.35468167066574097
175
- - 0.8703295588493347
176
- - 0.7531875371932983
177
- - 0.971237301826477
178
- - 0.13985762000083923
179
- - 0.15686897933483124
180
- - 0.4661160111427307
181
- - 0.7400000095367432
182
- - 0.5
183
- - 0.5
184
- - 1
185
- - 3.1414895057678223
186
- action_min:
187
- - 0
188
- - 0
189
- - 0
190
- - -1.5
191
- - -1.5
192
- - -0.6000000238418579
193
- - -1.5
194
- - -0.5
195
- - -0.699999988079071
196
- - -0.699999988079071
197
- - -2.216935230032842e-16
198
- - -2.216935230032842e-16
199
- - -4.0845591349633594e-18
200
- - -2.216935230032842e-16
201
- - -0.4883034825325012
202
- - 0.1900009959936142
203
- - -0.5470856428146362
204
- - -0.34318920969963074
205
- - -0.35952919721603394
206
- - -0.35302427411079407
207
- - -0.4469815790653229
208
- - -0.6371198296546936
209
- - -0.7683824300765991
210
- - -1.0653810501098633
211
- - -0.8479154706001282
212
- - -1.0297260284423828
213
- - -0.42936205863952637
214
- - -0.5147944092750549
215
- - -0.16820405423641205
216
- - -0.045328833162784576
217
- - -0.13282617926597595
218
- - 0.7400000095367432
219
- - -0.5
220
- - -0.5
221
- - -1
222
- - -3.138223648071289
223
- action_norm_masks:
224
- - true
225
- - true
226
- - true
227
- - true
228
- - true
229
- - true
230
- - false
231
- action_norm_type: bounds
232
- normalize_state: true
233
- pad_action_dim: 36
234
- pad_state_dim: 36
235
- stat_action_key: action
236
- stat_path: meta/stats_psi0.json
237
- stat_state_key: states
238
- state_max:
239
- - 0.47981399297714233
240
- - 0.6772664189338684
241
- - 0.6746510863304138
242
- - 0.0010172375477850437
243
- - 0.0007091082516126335
244
- - 0.001881288131698966
245
- - 0.0011398319620639086
246
- - 6.141255539660051e-07
247
- - 0.3043450713157654
248
- - 6.343479981296696e-07
249
- - 0.6933000087738037
250
- - 1.4612544775009155
251
- - 1.4651201963424683
252
- - 1.4609057903289795
253
- - 0.2809508740901947
254
- - 0.34028318524360657
255
- - 0.47627460956573486
256
- - 0.26476219296455383
257
- - 0.20825636386871338
258
- - 0.4566418528556824
259
- - 0.42864030599594116
260
- - 0.1656116098165512
261
- - -0.1549365073442459
262
- - 0.5154499411582947
263
- - 0.4242899715900421
264
- - 0.8548054695129395
265
- - 0.8040095567703247
266
- - 0.9811649322509766
267
- - 0.136736661195755
268
- - 0.195722296833992
269
- - 0.45781663060188293
270
- - 0.7400000095367432
271
- - 0
272
- - 0
273
- - 0
274
- - 0
275
- state_min:
276
- - -0.02442001923918724
277
- - -0.0517612099647522
278
- - -0.0006534014828503132
279
- - -0.5095356106758118
280
- - -1.323034405708313
281
- - -1.3221508264541626
282
- - -1.3230019807815552
283
- - -0.5770347714424133
284
- - -0.4338151812553406
285
- - -0.6721642017364502
286
- - -0.0017213862156495452
287
- - -7.534810038123396e-07
288
- - -0.001927333534695208
289
- - -1.075333216249419e-06
290
- - -0.43650975823402405
291
- - 0.15721464157104492
292
- - -0.5489339232444763
293
- - -0.2632291913032532
294
- - -0.3508843183517456
295
- - -0.23784859478473663
296
- - -0.4281824827194214
297
- - -0.5803383588790894
298
- - -0.7118590474128723
299
- - -1.0344431400299072
300
- - -0.7932196259498596
301
- - -1.0205217599868774
302
- - -0.3445618450641632
303
- - -0.5986371040344238
304
- - -0.13537253439426422
305
- - -0.0017330688424408436
306
- - -0.1421850621700287
307
- - 0.7400000095367432
308
- - 0
309
- - 0
310
- - 0
311
- - 0
312
- use_norm_mask: false
313
- model:
314
- center_crop:
315
- size:
316
- - 224
317
- - 224
318
- color_jitter:
319
- brightness: 0.2
320
- contrast:
321
- - 0.8
322
- - 1.2
323
- hue: 0.05
324
- saturation:
325
- - 0.8
326
- - 1.2
327
- img_aug: true
328
- normalize:
329
- mean:
330
- - 0.485
331
- - 0.456
332
- - 0.406
333
- std:
334
- - 0.229
335
- - 0.224
336
- - 0.225
337
- resize:
338
- size:
339
- - 256
340
- - 480
341
- repack:
342
- action_chunk_size: 30
343
- dataset_name: simple
344
- num_past_frames: 0
345
- pad_action_dim: 36
346
- pad_state_dim: 36
347
- val_repo_ids:
348
- - G1WholebodyLocomotionPickBetweenTablesTeleop-v0
349
- debug:
350
- value: false
351
- environment_variables:
352
- value:
353
- CUDA_VISIBLE_DEVICES: 4,5,6,7
354
- DATA_HOME: /data/data
355
- HF_HOME: /data/cache
356
- HF_LEROBOT_HOME: /data/data/lerobot
357
- HF_TOKEN: hf_...TiKa
358
- LOCAL_RANK: "0"
359
- LOCAL_WORLD_SIZE: "4"
360
- MASTER_ADDR: nebula99
361
- MASTER_PORT: "35899"
362
- OMP_NUM_THREADS: "32"
363
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
364
- RANK: "0"
365
- TORCH_HOME: /data/cache
366
- UV_CACHE_DIR: /data/cache
367
- WANDB_API_KEY: 90e...5c06
368
- WE_HOME: Not Set
369
- WORLD_SIZE: "4"
370
- eval:
371
- value: false
372
- exp:
373
- value: g1wholebodylocomotionpickbetweentablesteleop-v0
374
- log:
375
- value:
376
- log_freq: 100
377
- logging_dir: logs
378
- report_to: wandb
379
- model:
380
- value:
381
- action_dim: 36
382
- chunk_size: 30
383
- dim_feedforward: 3200
384
- dim_model: 512
385
- dropout: 0.1
386
- feedforward_activation: relu
387
- kl_weight: 10
388
- latent_dim: 32
389
- n_action_steps: 30
390
- n_decoder_layers: 1
391
- n_encoder_layers: 4
392
- n_heads: 8
393
- n_obs_steps: 1
394
- n_vae_encoder_layers: 4
395
- pre_norm: false
396
- state_dim: 36
397
- temporal_ensemble_coeff: null
398
- use_vae: true
399
- seed:
400
- value: 2026
401
- timestamp:
402
- value: "2604081550"
403
- train:
404
- value:
405
- checkpointing_steps: 5000
406
- data_parallel: ddp
407
- deepspeed_config: /data/jliu/psi/src/InternVLA/config/deepseeds/zero3.json
408
- enable_gradient_checkpointing: true
409
- enable_mixed_precision_training: true
410
- gradient_accumulation_steps: 1
411
- hf_token: .hf_token
412
- learning_rate: 0.0001
413
- lora: false
414
- lr_scheduler_kwargs:
415
- betas:
416
- - 0.95
417
- - 0.999
418
- eps: 1e-08
419
- weight_decay: 1e-06
420
- lr_scheduler_type: cosine
421
- max_checkpoints_to_keep: null
422
- max_grad_norm: 1
423
- max_training_steps: 40000
424
- mixed_precision: bf16
425
- name: act-g1
426
- num_train_epochs: null
427
- num_workers: 8
428
- optimizer_foreach: null
429
- output_dir: .runs
430
- overfit_single_batch: false
431
- reduce_in_full_precision: true
432
- resume_from_checkpoint: null
433
- sharding_strategy: full-shard
434
- skip_resumed_steps: false
435
- train_batch_size: 32
436
- val_batch_size: 16
437
- val_num_batches: 20
438
- validation_steps: 500
439
- warmup_ratio: null
440
- warmup_steps: 1000
441
- wandb:
442
- value:
443
- entity: jliu530-soochow-university
444
- group: null
445
- id: null
446
- name: null
447
- project: psi
448
- resume: allow
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/output.log DELETED
@@ -1,237 +0,0 @@
1
- [15:50:55 04/08] INFO  | >> [*] Saved configuration to ]8;id=805427;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=514750;file:///data/jliu/psi/scripts/train.py#128\128]8;;\
2
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550  
3
-   INFO  | >> [*] Training configurations: ]8;id=2657;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=644039;file:///data/jliu/psi/scripts/train.py#181\181]8;;\
4
-   INFO  | >> |=> training task: 'act-g1' ]8;id=471497;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=12046;file:///data/jliu/psi/scripts/train.py#182\182]8;;\
5
-   INFO  | >> |=> run name: ]8;id=329497;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=220436;file:///data/jliu/psi/scripts/train.py#183\183]8;;\
6
-   g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550  
7
-   INFO  | >> |=> seed: 2026 ]8;id=844246;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=394625;file:///data/jliu/psi/scripts/train.py#184\184]8;;\
8
-   INFO  | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=94020;file:///data/jliu/psi/scripts/train.py#185\185]8;;\
9
-   INFO  | >> |=> warmup steps: 1000 ]8;id=147732;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=919050;file:///data/jliu/psi/scripts/train.py#186\186]8;;\
10
-   INFO  | >> |=> validation steps: 500 ]8;id=25941;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=888976;file:///data/jliu/psi/scripts/train.py#187\187]8;;\
11
-   INFO  | >> |=> checkpoint steps: 5000 ]8;id=95089;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=418144;file:///data/jliu/psi/scripts/train.py#188\188]8;;\
12
-   INFO  | >> |=> max gradient norm: 1.0 ]8;id=122104;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=446109;file:///data/jliu/psi/scripts/train.py#189\189]8;;\
13
-   INFO  | >> PyTorch version 2.7.0 available. ]8;id=514883;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py\config.py]8;;\:]8;id=417488;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/datasets/config.py#54\54]8;;\
14
- Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████| 99/99 [00:00<00:00, 178442.67it/s]
15
- Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████| 99/99 [00:00<00:00, 200888.29it/s]
16
- [15:50:58 04/08] INFO  | >> |=> Num training samples: ]8;id=591396;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=971225;file:///data/jliu/psi/scripts/train.py#192\192]8;;\
17
-   INFO  | >> |=> Training dataset size: 62,764 ]8;id=603472;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=235431;file:///data/jliu/psi/scripts/train.py#193\193]8;;\
18
-   INFO  | >> |=> Val dataset size: 62,764 ]8;id=548632;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=929903;file:///data/jliu/psi/scripts/train.py#195\195]8;;\
19
-   INFO  | >> [*] Initialize optimizers and schedulers... ]8;id=492873;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=149330;file:///data/jliu/psi/scripts/train.py#198\198]8;;\
20
-   INFO  | >> [*] ***** Running training ***** ]8;id=127940;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=657111;file:///data/jliu/psi/scripts/train.py#202\202]8;;\
21
-   INFO  | >> |=> Num training examples = 62764 ]8;id=974614;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=572136;file:///data/jliu/psi/scripts/train.py#203\203]8;;\
22
-   INFO  | >> |=> Max training Epochs = 82 ]8;id=574775;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=838653;file:///data/jliu/psi/scripts/train.py#204\204]8;;\
23
-   INFO  | >> |=> Total optimization steps = 40000 ]8;id=927413;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=421473;file:///data/jliu/psi/scripts/train.py#205\205]8;;\
24
-   INFO  | >> |=> Num steps Per Epoch = 491 ]8;id=9727;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=34717;file:///data/jliu/psi/scripts/train.py#206\206]8;;\
25
-   INFO  | >> |=> Effective training epochs = 325.87 ]8;id=842580;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=548288;file:///data/jliu/psi/scripts/train.py#207\207]8;;\
26
-   INFO  | >> |=> Global train batch size (w. parallel, distributed & accumulation) = 128 ]8;id=272745;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=726293;file:///data/jliu/psi/scripts/train.py#208\208]8;;\
27
-   INFO  | >> |=> Device train batch size = 32 ]8;id=521307;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=847206;file:///data/jliu/psi/scripts/train.py#209\209]8;;\
28
-   INFO  | >> |=> Gradient Accumulation steps = 1 ]8;id=871789;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=304159;file:///data/jliu/psi/scripts/train.py#210\210]8;;\
29
-   INFO  | >> |=> Num processes (GPUs) = 4 ]8;id=459749;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=722818;file:///data/jliu/psi/scripts/train.py#211\211]8;;\
30
- [15:50:59 04/08] INFO  | >> [*] Accelerator runs in: ]8;id=966756;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=191940;file:///data/jliu/psi/scripts/train.py#218\218]8;;\
31
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550  
32
- Traing steps: 12%|███████▋ | 4999/40000 [27:49<2:56:12, 3.31it/s, loss=0.0669, lr=9.7e-05]
33
- [2026-04-08 15:51:05,959] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
34
- [15:51:06 04/08] INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -O3 -fPIC -fPIC -c ]8;id=680052;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=279409;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
35
-   /tmp/tmphf7av35s/test.c -o /tmp/tmphf7av35s/test.o  
36
-   INFO  | >> cc -pthread /tmp/tmphf7av35s/test.o -laio -o /tmp/tmphf7av35s/a.out ]8;id=204038;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=783864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
37
-   INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall ]8;id=960999;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=14714;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
38
-   -O3 -fPIC -fPIC -c /tmp/tmprste3cr8/test.c -o /tmp/tmprste3cr8/test.o  
39
-   INFO  | >> cc -pthread /tmp/tmprste3cr8/test.o -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o ]8;id=573141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=977459;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
40
-   /tmp/tmprste3cr8/a.out  
41
-   INFO  | >> cc -pthread -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall ]8;id=208448;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=259876;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
42
-   -O3 -fPIC -fPIC -c /tmp/tmp2xda_ky0/test.c -o /tmp/tmp2xda_ky0/test.o  
43
-   INFO  | >> cc -pthread /tmp/tmp2xda_ky0/test.o -laio -o /tmp/tmp2xda_ky0/a.out ]8;id=572992;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py\spawn.py]8;;\:]8;id=281580;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/setuptools/_distutils/spawn.py#77\77]8;;\
44
- [2026-04-08 15:51:07,435] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
45
-
46
- [16:18:49 04/08] INFO  | >> Saving current state to ]8;id=328668;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=696864;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
47
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000  
48
-   INFO  | >> Model weights saved in ]8;id=515887;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=723262;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
49
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000/mode  
50
-   l.safetensors  
51
- [16:18:50 04/08] INFO  | >> Optimizer state saved in ]8;id=495959;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=616494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
52
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000/opti  
53
-   mizer.bin  
54
-   INFO  | >> Scheduler state saved in ]8;id=304800;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=88534;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
55
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000/sche  
56
-   duler.bin  
57
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=821494;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=820417;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
58
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000/samp  
59
-   ler.bin  
60
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=432054;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=409371;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
61
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000/samp  
62
-   ler_1.bin  
63
-   INFO  | >> Random states saved in ]8;id=394271;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=14818;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
64
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000/rand  
65
-   om_states_0.pkl  
66
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_5000
67
- [16:45:48 04/08] INFO  | >> Saving current state to ]8;id=53870;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=874302;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
68
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000  
69
- [16:45:49 04/08] INFO  | >> Model weights saved in ]8;id=806914;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=990978;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
70
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000/mod  
71
-   el.safetensors  
72
- [16:45:50 04/08] INFO  | >> Optimizer state saved in ]8;id=343681;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=715961;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
73
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000/opt  
74
-   imizer.bin  
75
-   INFO  | >> Scheduler state saved in ]8;id=127268;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=521739;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
76
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000/sch  
77
-   eduler.bin  
78
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=844039;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=179027;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
79
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000/sam  
80
-   pler.bin  
81
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=454670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=653622;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
82
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000/sam  
83
-   pler_1.bin  
84
-   INFO  | >> Random states saved in ]8;id=711457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=745414;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
85
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000/ran  
86
-   dom_states_0.pkl  
87
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_10000
88
- [17:12:37 04/08] INFO  | >> Saving current state to ]8;id=204990;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=678670;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
89
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000  
90
- [17:12:38 04/08] INFO  | >> Model weights saved in ]8;id=571229;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=976178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
91
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000/mod  
92
-   el.safetensors  
93
-   INFO  | >> Optimizer state saved in ]8;id=791370;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=671982;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
94
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000/opt  
95
-   imizer.bin  
96
-   INFO  | >> Scheduler state saved in ]8;id=920457;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=21464;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
97
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000/sch  
98
-   eduler.bin  
99
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=823560;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=925542;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
100
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000/sam  
101
-   pler.bin  
102
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=693342;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=885391;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
103
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000/sam  
104
-   pler_1.bin  
105
-   INFO  | >> Random states saved in ]8;id=558902;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=434201;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
106
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000/ran  
107
-   dom_states_0.pkl  
108
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_15000
109
- [17:39:21 04/08] INFO  | >> Saving current state to ]8;id=196292;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=360779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
110
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000  
111
- [17:39:22 04/08] INFO  | >> Model weights saved in ]8;id=818149;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=854583;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
112
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000/mod  
113
-   el.safetensors  
114
- [17:39:23 04/08] INFO  | >> Optimizer state saved in ]8;id=341614;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=544562;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
115
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000/opt  
116
-   imizer.bin  
117
-   INFO  | >> Scheduler state saved in ]8;id=835298;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=550186;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
118
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000/sch  
119
-   eduler.bin  
120
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=889056;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=224742;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
121
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000/sam  
122
-   pler.bin  
123
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=921300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=476172;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
124
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000/sam  
125
-   pler_1.bin  
126
-   INFO  | >> Random states saved in ]8;id=821364;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=874045;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
127
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000/ran  
128
-   dom_states_0.pkl  
129
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_20000
130
- [18:06:10 04/08] INFO  | >> Saving current state to ]8;id=28505;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=369046;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
131
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000  
132
-   INFO  | >> Model weights saved in ]8;id=720536;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=600133;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
133
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000/mod  
134
-   el.safetensors  
135
- [18:06:11 04/08] INFO  | >> Optimizer state saved in ]8;id=650939;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=750793;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
136
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000/opt  
137
-   imizer.bin  
138
-   INFO  | >> Scheduler state saved in ]8;id=766478;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=224851;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
139
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000/sch  
140
-   eduler.bin  
141
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=702868;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=582729;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
142
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000/sam  
143
-   pler.bin  
144
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=101114;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=457234;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
145
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000/sam  
146
-   pler_1.bin  
147
-   INFO  | >> Random states saved in ]8;id=795987;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=373558;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
148
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000/ran  
149
-   dom_states_0.pkl  
150
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_25000
151
- [18:37:31 04/08] INFO  | >> Saving current state to ]8;id=660693;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=111484;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
152
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000  
153
- [18:37:32 04/08] INFO  | >> Model weights saved in ]8;id=878068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=67019;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
154
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000/mod  
155
-   el.safetensors  
156
- [18:37:33 04/08] INFO  | >> Optimizer state saved in ]8;id=273529;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=268565;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
157
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000/opt  
158
-   imizer.bin  
159
-   INFO  | >> Scheduler state saved in ]8;id=941677;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=948866;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
160
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000/sch  
161
-   eduler.bin  
162
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=60095;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=101135;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
163
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000/sam  
164
-   pler.bin  
165
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=98429;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=719141;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
166
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000/sam  
167
-   pler_1.bin  
168
-   INFO  | >> Random states saved in ]8;id=367323;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=330274;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
169
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000/ran  
170
-   dom_states_0.pkl  
171
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_30000
172
- [19:06:07 04/08] INFO  | >> Saving current state to ]8;id=686520;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=748910;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
173
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000  
174
-   INFO  | >> Model weights saved in ]8;id=331416;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=637309;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
175
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000/mod  
176
-   el.safetensors  
177
- [19:06:08 04/08] INFO  | >> Optimizer state saved in ]8;id=708752;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=921072;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
178
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000/opt  
179
-   imizer.bin  
180
-   INFO  | >> Scheduler state saved in ]8;id=525611;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=714503;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
181
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000/sch  
182
-   eduler.bin  
183
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=993068;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=856493;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
184
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000/sam  
185
-   pler.bin  
186
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=457956;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=921087;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
187
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000/sam  
188
-   pler_1.bin  
189
-   INFO  | >> Random states saved in ]8;id=303621;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=762447;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
190
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000/ran  
191
-   dom_states_0.pkl  
192
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_35000
193
- [19:34:21 04/08] INFO  | >> Saving current state to ]8;id=311366;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=805194;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
194
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000  
195
- [19:34:23 04/08] INFO  | >> Model weights saved in ]8;id=21254;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=568492;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
196
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/mod  
197
-   el.safetensors  
198
-   INFO  | >> Optimizer state saved in ]8;id=756734;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=118067;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
199
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/opt  
200
-   imizer.bin  
201
-   INFO  | >> Scheduler state saved in ]8;id=39131;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=131787;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
202
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/sch  
203
-   eduler.bin  
204
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=687655;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=316922;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
205
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/sam  
206
-   pler.bin  
207
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=373263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=435471;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
208
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/sam  
209
-   pler_1.bin  
210
-   INFO  | >> Random states saved in ]8;id=441435;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=694779;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
211
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000/ran  
212
-   dom_states_0.pkl  
213
- Saved state to .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_40000
214
- Training has reached maximum steps.
215
- [19:34:30 04/08] INFO  | >> Saving current state to ]8;id=569300;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=29797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/accelerator.py#3260\3260]8;;\
216
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_39999  
217
- [19:34:31 04/08] INFO  | >> Model weights saved in ]8;id=716966;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=581852;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#108\108]8;;\
218
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_39999/mod  
219
-   el.safetensors  
220
- [19:34:32 04/08] INFO  | >> Optimizer state saved in ]8;id=154857;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=239343;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#115\115]8;;\
221
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_39999/opt  
222
-   imizer.bin  
223
-   INFO  | >> Scheduler state saved in ]8;id=502103;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=438178;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#122\122]8;;\
224
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_39999/sch  
225
-   eduler.bin  
226
-   INFO  | >> Sampler state for dataloader 0 saved in ]8;id=657224;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=622487;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
227
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_39999/sam  
228
-   pler.bin  
229
-   INFO  | >> Sampler state for dataloader 1 saved in ]8;id=39855;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=681797;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#139\139]8;;\
230
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_39999/sam  
231
-   pler_1.bin  
232
-   INFO  | >> Random states saved in ]8;id=278106;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=357263;file:///data/jliu/psi/.venv-psi/lib/python3.10/site-packages/accelerate/checkpointing.py#170\170]8;;\
233
-   .runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/checkpoints/ckpt_39999/ran  
234
-   dom_states_0.pkl  
235
-   INFO  | >> [*] Finalized ACT Trainer. Epoch losses: [0.0, 1.9138156175613403, 0.7621236443519592, 0.3085829019546509, 0.21359018981456757, ]8;id=58988;file:///data/jliu/psi/src/psi/trainers/act_g1.py\act_g1.py]8;;\:]8;id=689824;file:///data/jliu/psi/src/psi/trainers/act_g1.py#331\331]8;;\
236
-   0.15654177963733673]  
237
-   INFO  | >> [*] Happy Ending! ]8;id=61128;file:///data/jliu/psi/scripts/train.py\train.py]8;;\:]8;id=232614;file:///data/jliu/psi/scripts/train.py#310\310]8;;\
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/requirements.txt DELETED
@@ -1,219 +0,0 @@
1
- tifffile==2025.5.10
2
- mpmath==1.3.0
3
- pynput==1.8.1
4
- exceptiongroup==1.3.0
5
- sympy==1.14.0
6
- decorator==5.2.1
7
- aiohttp-cors==0.8.1
8
- soupsieve==2.8
9
- Farama-Notifications==0.0.4
10
- numpydantic==1.6.7
11
- uvicorn==0.38.0
12
- waterbear==2.6.8
13
- comm==0.2.3
14
- cmake==4.2.3
15
- xxhash==3.5.0
16
- parso==0.8.6
17
- nvidia-cusparse-cu12==12.5.4.2
18
- annotated-doc==0.0.4
19
- jsonlines==4.0.0
20
- nvidia-cuda-runtime-cu12==12.6.77
21
- pydantic==2.10.6
22
- multidict==6.6.4
23
- shtab==1.7.2
24
- nvidia-cuda-nvrtc-cu12==12.6.77
25
- matplotlib-inline==0.2.1
26
- pycollada==0.9.2
27
- pure_eval==0.2.3
28
- beautifulsoup4==4.14.2
29
- msgspec==0.19.0
30
- huggingface-hub==0.35.3
31
- pydantic_core==2.27.2
32
- pytz==2025.2
33
- gymnasium==1.2.3
34
- attrs==25.3.0
35
- requests==2.32.5
36
- hf_transfer==0.1.9
37
- einx==0.3.0
38
- scipy==1.15.3
39
- transforms3d==0.4.2
40
- pyzmq==27.1.0
41
- aiohappyeyeballs==2.6.1
42
- six==1.17.0
43
- expandvars==1.1.2
44
- prompt_toolkit==3.0.52
45
- wcwidth==0.6.0
46
- stack-data==0.6.3
47
- ruamel.yaml.clib==0.2.14
48
- multiprocess==0.70.16
49
- vhacdx==0.0.9
50
- tokenizers==0.22.2
51
- aiosignal==1.4.0
52
- itsdangerous==2.2.0
53
- torchvision==0.22.0
54
- nvidia-cublas-cu12==12.6.4.1
55
- httpx==0.28.1
56
- platformdirs==4.4.0
57
- sentry-sdk==2.39.0
58
- plotly==6.2.0
59
- nvidia-cudnn-cu12==9.5.1.17
60
- jsonschema==4.25.1
61
- fastapi==0.119.1
62
- fsspec==2025.3.0
63
- MarkupSafe==3.0.3
64
- pydantic-yaml==1.6.0
65
- h11==0.16.0
66
- typer==0.24.1
67
- mypy_extensions==1.1.0
68
- nest-asyncio==1.6.0
69
- h5py==3.14.0
70
- python-xlib==0.33
71
- lazy_loader==0.4
72
- einops==0.8.1
73
- albumentations==1.4.18
74
- params_proto==2.13.2
75
- psutil==7.1.0
76
- starlette==0.48.0
77
- anyio==4.11.0
78
- gdown==5.2.0
79
- charset-normalizer==3.4.3
80
- tyro==0.9.32
81
- filelock==3.19.1
82
- websockets==15.0.1
83
- nvidia-cuda-cupti-cu12==12.6.80
84
- python-dotenv==1.2.1
85
- orderly-set==5.5.0
86
- inquirerpy==0.3.4
87
- pexpect==4.9.0
88
- ipython==8.38.0
89
- urllib3==2.5.0
90
- diffusers==0.37.0
91
- PyYAML==6.0.3
92
- antlr4-python3-runtime==4.9.3
93
- mdurl==0.1.2
94
- omegaconf==2.3.0
95
- rerun-sdk==0.22.1
96
- draccus==0.10.0
97
- sentencepiece==0.2.1
98
- referencing==0.37.0
99
- docstring_parser==0.17.0
100
- protobuf==6.33.5
101
- wandb==0.25.1
102
- numpy==1.26.4
103
- GitPython==3.1.45
104
- opencv-python-headless==4.11.0.86
105
- yourdfpy==0.0.58
106
- async-timeout==4.0.3
107
- shapely==2.1.2
108
- frozenlist==1.7.0
109
- simplejpeg==1.9.0
110
- asttokens==3.0.1
111
- Pygments==2.19.2
112
- py-cpuinfo==9.0.0
113
- rtree==1.4.1
114
- jedi==0.19.2
115
- gitdb==4.0.12
116
- cloudpickle==3.1.1
117
- deepspeed==0.17.1
118
- executing==2.2.1
119
- nvidia-nvjitlink-cu12==12.6.85
120
- colorlog==6.10.1
121
- qwen-vl-utils==0.0.14
122
- Werkzeug==3.1.6
123
- zipp==3.23.0
124
- setuptools==80.9.0
125
- albucore==0.0.17
126
- transformers==4.57.0
127
- imageio==2.34.2
128
- frozendict==2.4.6
129
- hjson==3.1.0
130
- jsonschema-specifications==2025.9.1
131
- rpds-py==0.28.0
132
- tornado==6.5.5
133
- blinker==1.9.0
134
- accelerate==1.7.0
135
- argparse==1.4.0
136
- msgpack==1.1.1
137
- smmap==5.0.2
138
- nvidia-nvtx-cu12==12.6.77
139
- packaging==25.0
140
- embreex==2.17.7.post7
141
- vuer==0.0.68
142
- propcache==0.3.2
143
- nvidia-cusolver-cu12==11.7.1.2
144
- triton==3.3.0
145
- pfzy==0.3.4
146
- debugpy==1.8.20
147
- torchcodec==0.4.0
148
- dm-tree==0.1.8
149
- ninja==1.13.0
150
- ipykernel==7.2.0
151
- traitlets==5.14.3
152
- nvidia-cufile-cu12==1.11.1.6
153
- datasets==3.6.0
154
- nvidia-curand-cu12==10.3.7.77
155
- flash_attn==2.7.4.post1
156
- av==16.0.1
157
- sniffio==1.3.1
158
- Jinja2==3.1.6
159
- narwhals==2.6.0
160
- yarl==1.20.1
161
- typeguard==4.4.4
162
- termcolor==3.1.0
163
- jupyter_client==8.8.0
164
- importlib_metadata==8.7.1
165
- eval_type_backport==0.2.2
166
- certifi==2025.8.3
167
- tzdata==2025.2
168
- typing_extensions==4.15.0
169
- viser==1.0.15
170
- typing-inspect==0.9.0
171
- annotated-types==0.7.0
172
- svg.path==7.0
173
- mergedeep==1.3.4
174
- ptyprocess==0.7.0
175
- psi==0.0.0
176
- deepdiff==8.6.1
177
- toml==0.10.2
178
- click==8.3.0
179
- dotenv==0.9.9
180
- trimesh==4.8.3
181
- nvidia-nccl-cu12==2.26.2
182
- pyyaml-include==1.4.1
183
- scikit-image==0.25.2
184
- PySocks==1.7.1
185
- aiohttp==3.10.5
186
- opencv-python==4.11.0.86
187
- nodeenv==1.9.1
188
- lerobot==0.3.3
189
- pillow==11.3.0
190
- idna==3.10
191
- Flask==3.1.3
192
- killport==1.2.0
193
- hf-xet==1.1.10
194
- imageio-ffmpeg==0.6.0
195
- httpcore==1.0.9
196
- nvidia-cusparselt-cu12==0.6.3
197
- networkx==3.4.2
198
- ruamel.yaml==0.18.15
199
- manifold3d==3.2.1
200
- shellingham==1.5.4
201
- nvidia-cufft-cu12==11.3.0.4
202
- mapbox_earcut==1.0.3
203
- peft==0.17.1
204
- pyarrow==21.0.0
205
- pyserial==3.5
206
- torch==2.7.0
207
- dill==0.3.8
208
- markdown-it-py==4.0.0
209
- tqdm==4.67.1
210
- argcomplete==3.6.2
211
- jupyter_core==5.9.1
212
- pandas==2.3.3
213
- lxml==6.0.2
214
- safetensors==0.6.2
215
- python-dateutil==2.9.0.post0
216
- regex==2025.9.18
217
- evdev==1.9.3
218
- rich==14.1.0
219
- psi==0.0.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
act/act-g1-sim/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550/wandb/run-20260408_155054-ii0lixdx/files/wandb-metadata.json DELETED
@@ -1,132 +0,0 @@
1
- {
2
- "os": "Linux-6.8.0-106-generic-x86_64-with-glibc2.39",
3
- "python": "CPython 3.10.20",
4
- "startedAt": "2026-04-08T15:50:54.514093Z",
5
- "args": [
6
- "simple_act_config",
7
- "--seed=2026",
8
- "--exp=g1wholebodylocomotionpickbetweentablesteleop-v0",
9
- "--train.name=act-g1",
10
- "--log.report-to=wandb",
11
- "--train.data_parallel=ddp",
12
- "--train.mixed_precision=bf16",
13
- "--train.train-batch-size=32",
14
- "--train.warmup-steps=1000",
15
- "--train.warmup-ratio=None",
16
- "--train.checkpointing-steps=5000",
17
- "--train.validation_steps=500",
18
- "--train.val_num_batches=20",
19
- "--train.gradient_accumulation_steps=1",
20
- "--train.max-training-steps=40000",
21
- "--train.learning-rate=1e-4",
22
- "--train.max-grad-norm=1.0",
23
- "--train.lr_scheduler_kwargs.weight_decay=1e-6",
24
- "--train.lr_scheduler_kwargs.betas",
25
- "0.95",
26
- "0.999",
27
- "--train.lr_scheduler_type=cosine",
28
- "--data.root_dir=/data/jliu/data",
29
- "--data.train-repo-ids=G1WholebodyLocomotionPickBetweenTablesTeleop-v0",
30
- "--data.transform.repack.action_chunk_size=30",
31
- "--data.transform.repack.pad-action-dim=36",
32
- "--data.transform.repack.pad-state-dim=36",
33
- "--data.transform.field.stat-path=meta/stats_psi0.json",
34
- "--data.transform.field.stat-action-key=action",
35
- "--data.transform.field.stat-state-key=states",
36
- "--data.transform.field.normalize-state",
37
- "--data.transform.field.action-norm-type=bounds",
38
- "--data.transform.field.pad-action-dim=36",
39
- "--data.transform.field.pad-state-dim=36",
40
- "--data.transform.model.img-aug",
41
- "--model.chunk-size=30",
42
- "--model.n-action-steps=30",
43
- "--model.action-dim=36",
44
- "--model.state-dim=36",
45
- "--model.use-vae",
46
- "--model.kl-weight=10.0"
47
- ],
48
- "program": "/data/jliu/psi/scripts/train.py",
49
- "codePath": "scripts/train.py",
50
- "codePathLocal": "scripts/train.py",
51
- "git": {
52
- "remote": "https://github.com/songlin/psi.git",
53
- "commit": "04614628ecb677f5f278e2d31c1103aed8127f26"
54
- },
55
- "email": "jliu530@163.com",
56
- "root": "/data/jliu/psi/.runs/act-g1/g1wholebodylocomotionpickbetweentablesteleop-v0.g1.cosine.lr1.0e-04.b128.gpus4.2604081550",
57
- "host": "nebula99",
58
- "executable": "/data/jliu/psi/.venv-psi/bin/python3",
59
- "cpu_count": 128,
60
- "cpu_count_logical": 128,
61
- "gpu": "NVIDIA A100-SXM4-80GB",
62
- "gpu_count": 8,
63
- "disk": {
64
- "/": {
65
- "total": "105089261568",
66
- "used": "92984258560"
67
- }
68
- },
69
- "memory": {
70
- "total": "1623177445376"
71
- },
72
- "gpu_nvidia": [
73
- {
74
- "name": "NVIDIA A100-SXM4-80GB",
75
- "memoryTotal": "85899345920",
76
- "cudaCores": 6912,
77
- "architecture": "Ampere",
78
- "uuid": "GPU-02b51758-2aec-8c71-c751-92567fdb15ad"
79
- },
80
- {
81
- "name": "NVIDIA A100-SXM4-80GB",
82
- "memoryTotal": "85899345920",
83
- "cudaCores": 6912,
84
- "architecture": "Ampere",
85
- "uuid": "GPU-be645bb4-1ef0-15da-fc6b-53c5378650df"
86
- },
87
- {
88
- "name": "NVIDIA A100-SXM4-80GB",
89
- "memoryTotal": "85899345920",
90
- "cudaCores": 6912,
91
- "architecture": "Ampere",
92
- "uuid": "GPU-44868ccd-5809-61e2-f8cc-1dff8efc7eb1"
93
- },
94
- {
95
- "name": "NVIDIA A100-SXM4-80GB",
96
- "memoryTotal": "85899345920",
97
- "cudaCores": 6912,
98
- "architecture": "Ampere",
99
- "uuid": "GPU-a12b8de3-fb12-2901-c3a2-1e0e2b050b7b"
100
- },
101
- {
102
- "name": "NVIDIA A100-SXM4-80GB",
103
- "memoryTotal": "85899345920",
104
- "cudaCores": 6912,
105
- "architecture": "Ampere",
106
- "uuid": "GPU-57787183-4904-b2c9-cf52-1c1af976b365"
107
- },
108
- {
109
- "name": "NVIDIA A100-SXM4-80GB",
110
- "memoryTotal": "85899345920",
111
- "cudaCores": 6912,
112
- "architecture": "Ampere",
113
- "uuid": "GPU-e82d0fd0-9dcb-3d7c-9451-c4770ad6ddc3"
114
- },
115
- {
116
- "name": "NVIDIA A100-SXM4-80GB",
117
- "memoryTotal": "85899345920",
118
- "cudaCores": 6912,
119
- "architecture": "Ampere",
120
- "uuid": "GPU-9a825fa3-60d0-3acf-6c79-67e9230ae24c"
121
- },
122
- {
123
- "name": "NVIDIA A100-SXM4-80GB",
124
- "memoryTotal": "85899345920",
125
- "cudaCores": 6912,
126
- "architecture": "Ampere",
127
- "uuid": "GPU-6fe02e5e-b886-9c02-e433-6237f3897dd8"
128
- }
129
- ],
130
- "cudaVersion": "12.9",
131
- "writerId": "ufhcsemgf4luglllwa36q76bqs2rc5qh"
132
- }