rguntz commited on
Commit
5a488e7
·
verified ·
1 Parent(s): 9e8ef5f

Upload folder using huggingface_hub

Browse files
Files changed (43) hide show
  1. checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/config.json +92 -0
  2. checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/model.safetensors +3 -0
  3. checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_postprocessor.json +32 -0
  4. checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors +3 -0
  5. checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_preprocessor.json +56 -0
  6. checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors +3 -0
  7. checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/train_config.json +302 -0
  8. checkpoints/dagger/1iter/checkpoints/018000/training_state/optimizer_param_groups.json +396 -0
  9. checkpoints/dagger/1iter/checkpoints/018000/training_state/optimizer_state.safetensors +3 -0
  10. checkpoints/dagger/1iter/checkpoints/018000/training_state/rng_state.safetensors +3 -0
  11. checkpoints/dagger/1iter/checkpoints/018000/training_state/scheduler_state.json +15 -0
  12. checkpoints/dagger/1iter/checkpoints/018000/training_state/training_step.json +3 -0
  13. checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/config.json +92 -0
  14. checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/model.safetensors +3 -0
  15. checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_postprocessor.json +32 -0
  16. checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors +3 -0
  17. checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_preprocessor.json +56 -0
  18. checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors +3 -0
  19. checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/train_config.json +302 -0
  20. checkpoints/dagger/1iter/checkpoints/020000/training_state/optimizer_param_groups.json +396 -0
  21. checkpoints/dagger/1iter/checkpoints/020000/training_state/optimizer_state.safetensors +3 -0
  22. checkpoints/dagger/1iter/checkpoints/020000/training_state/rng_state.safetensors +3 -0
  23. checkpoints/dagger/1iter/checkpoints/020000/training_state/scheduler_state.json +15 -0
  24. checkpoints/dagger/1iter/checkpoints/020000/training_state/training_step.json +3 -0
  25. checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/config.json +92 -0
  26. checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/model.safetensors +3 -0
  27. checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_postprocessor.json +32 -0
  28. checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors +3 -0
  29. checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_preprocessor.json +56 -0
  30. checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors +3 -0
  31. checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/train_config.json +302 -0
  32. checkpoints/dagger/1iter/checkpoints/022000/training_state/optimizer_state.safetensors +3 -0
  33. checkpoints/dagger/1iter/checkpoints/022000/training_state/rng_state.safetensors +3 -0
  34. checkpoints/dagger/1iter/checkpoints/022000/training_state/training_step.json +3 -0
  35. checkpoints/dagger/1iter/wandb/debug-internal.log +79 -0
  36. checkpoints/dagger/1iter/wandb/debug.log +15 -0
  37. checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/files/config.yaml +0 -0
  38. checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/files/output.log +265 -1
  39. checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/files/wandb-summary.json +0 -0
  40. checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/logs/debug-core.log +8 -0
  41. checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/logs/debug-internal.log +79 -0
  42. checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/logs/debug.log +15 -0
  43. checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/run-ql0pvhal.wandb +2 -2
checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/config.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "diffusion",
3
+ "n_obs_steps": 2,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 6
9
+ ]
10
+ },
11
+ "observation.images.front": {
12
+ "type": "VISUAL",
13
+ "shape": [
14
+ 3,
15
+ 480,
16
+ 640
17
+ ]
18
+ }
19
+ },
20
+ "output_features": {
21
+ "action": {
22
+ "type": "ACTION",
23
+ "shape": [
24
+ 6
25
+ ]
26
+ }
27
+ },
28
+ "device": "cuda",
29
+ "use_amp": false,
30
+ "use_peft": false,
31
+ "push_to_hub": false,
32
+ "repo_id": null,
33
+ "private": null,
34
+ "tags": null,
35
+ "license": null,
36
+ "pretrained_path": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_specialized_523_dataset_base_from_dino_780_new_config_11H47/checkpoints/010000/pretrained_model",
37
+ "horizon": 16,
38
+ "n_action_steps": 8,
39
+ "normalization_mapping": {
40
+ "VISUAL": "MEAN_STD",
41
+ "STATE": "MIN_MAX",
42
+ "ACTION": "MIN_MAX"
43
+ },
44
+ "drop_n_last_frames": 7,
45
+ "vision_backbone_type": "dinov2",
46
+ "vision_encoder_name": "facebook/dinov2-small",
47
+ "freeze_vision_encoder": true,
48
+ "vision_backbone": "resnet18",
49
+ "resize_shape": [
50
+ 224,
51
+ 224
52
+ ],
53
+ "crop_ratio": 1.0,
54
+ "crop_shape": null,
55
+ "crop_is_random": true,
56
+ "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
57
+ "use_group_norm": false,
58
+ "spatial_softmax_num_keypoints": 32,
59
+ "vit_pool_type": "spatial_softmax",
60
+ "vit_feature_dim": 512,
61
+ "use_separate_rgb_encoder_per_camera": true,
62
+ "down_dims": [
63
+ 512,
64
+ 1024,
65
+ 2048
66
+ ],
67
+ "kernel_size": 5,
68
+ "n_groups": 8,
69
+ "diffusion_step_embed_dim": 128,
70
+ "use_film_scale_modulation": true,
71
+ "noise_scheduler_type": "DDPM",
72
+ "num_train_timesteps": 100,
73
+ "beta_schedule": "squaredcos_cap_v2",
74
+ "beta_start": 0.0001,
75
+ "beta_end": 0.02,
76
+ "prediction_type": "epsilon",
77
+ "clip_sample": true,
78
+ "clip_sample_range": 1.0,
79
+ "num_inference_steps": 100,
80
+ "compile_model": false,
81
+ "compile_mode": "reduce-overhead",
82
+ "do_mask_loss_for_padding": false,
83
+ "optimizer_lr": 5e-05,
84
+ "optimizer_betas": [
85
+ 0.95,
86
+ 0.999
87
+ ],
88
+ "optimizer_eps": 1e-08,
89
+ "optimizer_weight_decay": 1e-06,
90
+ "scheduler_name": "cosine",
91
+ "scheduler_warmup_steps": 100
92
+ }
checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e15da4c83e030433838104f06891be27b45b5558ed4815edc4631021e8d1113
3
+ size 1095367712
checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_postprocessor.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "unnormalizer_processor",
6
+ "config": {
7
+ "eps": 1e-08,
8
+ "features": {
9
+ "action": {
10
+ "type": "ACTION",
11
+ "shape": [
12
+ 6
13
+ ]
14
+ }
15
+ },
16
+ "norm_map": {
17
+ "VISUAL": "MEAN_STD",
18
+ "STATE": "MIN_MAX",
19
+ "ACTION": "MIN_MAX"
20
+ }
21
+ },
22
+ "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
23
+ },
24
+ {
25
+ "registry_name": "device_processor",
26
+ "config": {
27
+ "device": "cpu",
28
+ "float_dtype": null
29
+ }
30
+ }
31
+ ]
32
+ }
checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2cffd05b8fcb092f386f0047017f12099a576aa260612cfb4a12f36d9746bfa
3
+ size 6560
checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_preprocessor.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {}
8
+ }
9
+ },
10
+ {
11
+ "registry_name": "to_batch_processor",
12
+ "config": {}
13
+ },
14
+ {
15
+ "registry_name": "device_processor",
16
+ "config": {
17
+ "device": "cuda",
18
+ "float_dtype": null
19
+ }
20
+ },
21
+ {
22
+ "registry_name": "normalizer_processor",
23
+ "config": {
24
+ "eps": 1e-08,
25
+ "features": {
26
+ "observation.state": {
27
+ "type": "STATE",
28
+ "shape": [
29
+ 6
30
+ ]
31
+ },
32
+ "observation.images.front": {
33
+ "type": "VISUAL",
34
+ "shape": [
35
+ 3,
36
+ 480,
37
+ 640
38
+ ]
39
+ },
40
+ "action": {
41
+ "type": "ACTION",
42
+ "shape": [
43
+ 6
44
+ ]
45
+ }
46
+ },
47
+ "norm_map": {
48
+ "VISUAL": "MEAN_STD",
49
+ "STATE": "MIN_MAX",
50
+ "ACTION": "MIN_MAX"
51
+ }
52
+ },
53
+ "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors"
54
+ }
55
+ ]
56
+ }
checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2cffd05b8fcb092f386f0047017f12099a576aa260612cfb4a12f36d9746bfa
3
+ size 6560
checkpoints/dagger/1iter/checkpoints/018000/pretrained_model/train_config.json ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "local/dagger_episodes",
4
+ "root": "/home/shadeform/Desktop/robot_learning/base_new_dataset_dagger_628",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "same_cloth_color": false,
11
+ "tfs": {
12
+ "color_jitter": {
13
+ "weight": 1.5,
14
+ "type": "ColorJitter",
15
+ "kwargs": {
16
+ "brightness": [
17
+ 0.7,
18
+ 1.3
19
+ ],
20
+ "contrast": [
21
+ 0.7,
22
+ 1.3
23
+ ],
24
+ "saturation": [
25
+ 0.4,
26
+ 1.6
27
+ ],
28
+ "hue": [
29
+ -0.5,
30
+ 0.5
31
+ ]
32
+ }
33
+ },
34
+ "grayscale": {
35
+ "weight": 0.5,
36
+ "type": "RandomGrayscale",
37
+ "kwargs": {
38
+ "p": 1.0
39
+ }
40
+ },
41
+ "shadow": {
42
+ "weight": 0.8,
43
+ "type": "RandomShadow",
44
+ "kwargs": {
45
+ "num_shadows": [
46
+ 1,
47
+ 2
48
+ ],
49
+ "intensity": [
50
+ 0.4,
51
+ 0.7
52
+ ],
53
+ "blur_sigma": [
54
+ 4.0,
55
+ 20.0
56
+ ]
57
+ }
58
+ },
59
+ "highlight": {
60
+ "weight": 0.5,
61
+ "type": "RandomHighlight",
62
+ "kwargs": {
63
+ "num_highlights": [
64
+ 0,
65
+ 1
66
+ ],
67
+ "intensity": [
68
+ 0.2,
69
+ 0.5
70
+ ],
71
+ "sigma": [
72
+ 20.0,
73
+ 60.0
74
+ ]
75
+ }
76
+ },
77
+ "gamma": {
78
+ "weight": 0.7,
79
+ "type": "RandomGamma",
80
+ "kwargs": {
81
+ "gamma": [
82
+ 0.6,
83
+ 1.4
84
+ ]
85
+ }
86
+ },
87
+ "blur": {
88
+ "weight": 0.0,
89
+ "type": "GaussianBlur",
90
+ "kwargs": {
91
+ "kernel_size": [
92
+ 3,
93
+ 3
94
+ ],
95
+ "sigma": [
96
+ 0.1,
97
+ 1.5
98
+ ]
99
+ }
100
+ },
101
+ "jpeg": {
102
+ "weight": 0.0,
103
+ "type": "RandomJPEG",
104
+ "kwargs": {
105
+ "quality": [
106
+ 60,
107
+ 95
108
+ ]
109
+ }
110
+ },
111
+ "crop": {
112
+ "weight": 0.5,
113
+ "type": "RandomCropPreserveSize",
114
+ "kwargs": {
115
+ "scale": [
116
+ 0.97,
117
+ 1.0
118
+ ],
119
+ "ratio": [
120
+ 0.99,
121
+ 1.01
122
+ ]
123
+ }
124
+ },
125
+ "affine": {
126
+ "weight": 0.0,
127
+ "type": "RandomAffine",
128
+ "kwargs": {
129
+ "degrees": [
130
+ -5.0,
131
+ 5.0
132
+ ],
133
+ "translate": [
134
+ 0.05,
135
+ 0.05
136
+ ]
137
+ }
138
+ },
139
+ "sharpness": {
140
+ "weight": 1.0,
141
+ "type": "SharpnessJitter",
142
+ "kwargs": {
143
+ "sharpness": [
144
+ 0.5,
145
+ 1.5
146
+ ]
147
+ }
148
+ }
149
+ }
150
+ },
151
+ "revision": null,
152
+ "use_imagenet_stats": true,
153
+ "video_backend": "pyav",
154
+ "return_uint8": false,
155
+ "streaming": false
156
+ },
157
+ "env": null,
158
+ "policy": {
159
+ "type": "diffusion",
160
+ "n_obs_steps": 2,
161
+ "input_features": {
162
+ "observation.state": {
163
+ "type": "STATE",
164
+ "shape": [
165
+ 6
166
+ ]
167
+ },
168
+ "observation.images.front": {
169
+ "type": "VISUAL",
170
+ "shape": [
171
+ 3,
172
+ 480,
173
+ 640
174
+ ]
175
+ }
176
+ },
177
+ "output_features": {
178
+ "action": {
179
+ "type": "ACTION",
180
+ "shape": [
181
+ 6
182
+ ]
183
+ }
184
+ },
185
+ "device": "cuda",
186
+ "use_amp": false,
187
+ "use_peft": false,
188
+ "push_to_hub": false,
189
+ "repo_id": null,
190
+ "private": null,
191
+ "tags": null,
192
+ "license": null,
193
+ "pretrained_path": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_specialized_523_dataset_base_from_dino_780_new_config_11H47/checkpoints/010000/pretrained_model",
194
+ "horizon": 16,
195
+ "n_action_steps": 8,
196
+ "normalization_mapping": {
197
+ "VISUAL": "MEAN_STD",
198
+ "STATE": "MIN_MAX",
199
+ "ACTION": "MIN_MAX"
200
+ },
201
+ "drop_n_last_frames": 7,
202
+ "vision_backbone_type": "dinov2",
203
+ "vision_encoder_name": "facebook/dinov2-small",
204
+ "freeze_vision_encoder": true,
205
+ "vision_backbone": "resnet18",
206
+ "resize_shape": [
207
+ 224,
208
+ 224
209
+ ],
210
+ "crop_ratio": 1.0,
211
+ "crop_shape": null,
212
+ "crop_is_random": true,
213
+ "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
214
+ "use_group_norm": false,
215
+ "spatial_softmax_num_keypoints": 32,
216
+ "vit_pool_type": "spatial_softmax",
217
+ "vit_feature_dim": 512,
218
+ "use_separate_rgb_encoder_per_camera": true,
219
+ "down_dims": [
220
+ 512,
221
+ 1024,
222
+ 2048
223
+ ],
224
+ "kernel_size": 5,
225
+ "n_groups": 8,
226
+ "diffusion_step_embed_dim": 128,
227
+ "use_film_scale_modulation": true,
228
+ "noise_scheduler_type": "DDPM",
229
+ "num_train_timesteps": 100,
230
+ "beta_schedule": "squaredcos_cap_v2",
231
+ "beta_start": 0.0001,
232
+ "beta_end": 0.02,
233
+ "prediction_type": "epsilon",
234
+ "clip_sample": true,
235
+ "clip_sample_range": 1.0,
236
+ "num_inference_steps": 100,
237
+ "compile_model": false,
238
+ "compile_mode": "reduce-overhead",
239
+ "do_mask_loss_for_padding": false,
240
+ "optimizer_lr": 5e-05,
241
+ "optimizer_betas": [
242
+ 0.95,
243
+ 0.999
244
+ ],
245
+ "optimizer_eps": 1e-08,
246
+ "optimizer_weight_decay": 1e-06,
247
+ "scheduler_name": "cosine",
248
+ "scheduler_warmup_steps": 100
249
+ },
250
+ "reward_model": null,
251
+ "output_dir": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_dagger_iter1_25000steps_20260520_173844",
252
+ "job_name": "diffusion_dagger_iter1_25000steps_20260520_173844",
253
+ "resume": false,
254
+ "seed": 1000,
255
+ "cudnn_deterministic": false,
256
+ "num_workers": 16,
257
+ "batch_size": 64,
258
+ "prefetch_factor": 4,
259
+ "persistent_workers": true,
260
+ "steps": 25000,
261
+ "eval_freq": 0,
262
+ "log_freq": 20,
263
+ "tolerance_s": 0.0001,
264
+ "save_checkpoint": true,
265
+ "save_freq": 2000,
266
+ "use_policy_training_preset": true,
267
+ "optimizer": {
268
+ "type": "adam",
269
+ "lr": 5e-05,
270
+ "weight_decay": 1e-06,
271
+ "grad_clip_norm": 10.0,
272
+ "betas": [
273
+ 0.95,
274
+ 0.999
275
+ ],
276
+ "eps": 1e-08
277
+ },
278
+ "scheduler": {
279
+ "type": "diffuser",
280
+ "num_warmup_steps": 100,
281
+ "name": "cosine"
282
+ },
283
+ "eval": {
284
+ "n_episodes": 50,
285
+ "batch_size": 19,
286
+ "use_async_envs": true
287
+ },
288
+ "wandb": {
289
+ "enable": true,
290
+ "disable_artifact": false,
291
+ "project": "lerobot",
292
+ "entity": null,
293
+ "notes": null,
294
+ "run_id": "ql0pvhal",
295
+ "mode": null,
296
+ "add_tags": true
297
+ },
298
+ "peft": null,
299
+ "sample_weighting": null,
300
+ "rename_map": {},
301
+ "checkpoint_path": null
302
+ }
checkpoints/dagger/1iter/checkpoints/018000/training_state/optimizer_param_groups.json ADDED
@@ -0,0 +1,396 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "lr": 9.132549638500307e-06,
4
+ "betas": [
5
+ 0.95,
6
+ 0.999
7
+ ],
8
+ "eps": 1e-08,
9
+ "weight_decay": 1e-06,
10
+ "amsgrad": false,
11
+ "maximize": false,
12
+ "foreach": null,
13
+ "capturable": false,
14
+ "differentiable": false,
15
+ "fused": null,
16
+ "decoupled_weight_decay": false,
17
+ "initial_lr": 5e-05,
18
+ "params": [
19
+ 0,
20
+ 1,
21
+ 2,
22
+ 3,
23
+ 4,
24
+ 5,
25
+ 6,
26
+ 7,
27
+ 8,
28
+ 9,
29
+ 10,
30
+ 11,
31
+ 12,
32
+ 13,
33
+ 14,
34
+ 15,
35
+ 16,
36
+ 17,
37
+ 18,
38
+ 19,
39
+ 20,
40
+ 21,
41
+ 22,
42
+ 23,
43
+ 24,
44
+ 25,
45
+ 26,
46
+ 27,
47
+ 28,
48
+ 29,
49
+ 30,
50
+ 31,
51
+ 32,
52
+ 33,
53
+ 34,
54
+ 35,
55
+ 36,
56
+ 37,
57
+ 38,
58
+ 39,
59
+ 40,
60
+ 41,
61
+ 42,
62
+ 43,
63
+ 44,
64
+ 45,
65
+ 46,
66
+ 47,
67
+ 48,
68
+ 49,
69
+ 50,
70
+ 51,
71
+ 52,
72
+ 53,
73
+ 54,
74
+ 55,
75
+ 56,
76
+ 57,
77
+ 58,
78
+ 59,
79
+ 60,
80
+ 61,
81
+ 62,
82
+ 63,
83
+ 64,
84
+ 65,
85
+ 66,
86
+ 67,
87
+ 68,
88
+ 69,
89
+ 70,
90
+ 71,
91
+ 72,
92
+ 73,
93
+ 74,
94
+ 75,
95
+ 76,
96
+ 77,
97
+ 78,
98
+ 79,
99
+ 80,
100
+ 81,
101
+ 82,
102
+ 83,
103
+ 84,
104
+ 85,
105
+ 86,
106
+ 87,
107
+ 88,
108
+ 89,
109
+ 90,
110
+ 91,
111
+ 92,
112
+ 93,
113
+ 94,
114
+ 95,
115
+ 96,
116
+ 97,
117
+ 98,
118
+ 99,
119
+ 100,
120
+ 101,
121
+ 102,
122
+ 103,
123
+ 104,
124
+ 105,
125
+ 106,
126
+ 107,
127
+ 108,
128
+ 109,
129
+ 110,
130
+ 111,
131
+ 112,
132
+ 113,
133
+ 114,
134
+ 115,
135
+ 116,
136
+ 117,
137
+ 118,
138
+ 119,
139
+ 120,
140
+ 121,
141
+ 122,
142
+ 123,
143
+ 124,
144
+ 125,
145
+ 126,
146
+ 127,
147
+ 128,
148
+ 129,
149
+ 130,
150
+ 131,
151
+ 132,
152
+ 133,
153
+ 134,
154
+ 135,
155
+ 136,
156
+ 137,
157
+ 138,
158
+ 139,
159
+ 140,
160
+ 141,
161
+ 142,
162
+ 143,
163
+ 144,
164
+ 145,
165
+ 146,
166
+ 147,
167
+ 148,
168
+ 149,
169
+ 150,
170
+ 151,
171
+ 152,
172
+ 153,
173
+ 154,
174
+ 155,
175
+ 156,
176
+ 157,
177
+ 158,
178
+ 159,
179
+ 160,
180
+ 161,
181
+ 162,
182
+ 163,
183
+ 164,
184
+ 165,
185
+ 166,
186
+ 167,
187
+ 168,
188
+ 169,
189
+ 170,
190
+ 171,
191
+ 172,
192
+ 173,
193
+ 174,
194
+ 175,
195
+ 176,
196
+ 177,
197
+ 178,
198
+ 179,
199
+ 180,
200
+ 181,
201
+ 182,
202
+ 183,
203
+ 184,
204
+ 185,
205
+ 186,
206
+ 187,
207
+ 188,
208
+ 189,
209
+ 190,
210
+ 191,
211
+ 192,
212
+ 193,
213
+ 194,
214
+ 195,
215
+ 196,
216
+ 197,
217
+ 198,
218
+ 199,
219
+ 200,
220
+ 201,
221
+ 202,
222
+ 203,
223
+ 204,
224
+ 205,
225
+ 206,
226
+ 207,
227
+ 208,
228
+ 209,
229
+ 210,
230
+ 211,
231
+ 212,
232
+ 213,
233
+ 214,
234
+ 215,
235
+ 216,
236
+ 217,
237
+ 218,
238
+ 219,
239
+ 220,
240
+ 221,
241
+ 222,
242
+ 223,
243
+ 224,
244
+ 225,
245
+ 226,
246
+ 227,
247
+ 228,
248
+ 229,
249
+ 230,
250
+ 231,
251
+ 232,
252
+ 233,
253
+ 234,
254
+ 235,
255
+ 236,
256
+ 237,
257
+ 238,
258
+ 239,
259
+ 240,
260
+ 241,
261
+ 242,
262
+ 243,
263
+ 244,
264
+ 245,
265
+ 246,
266
+ 247,
267
+ 248,
268
+ 249,
269
+ 250,
270
+ 251,
271
+ 252,
272
+ 253,
273
+ 254,
274
+ 255,
275
+ 256,
276
+ 257,
277
+ 258,
278
+ 259,
279
+ 260,
280
+ 261,
281
+ 262,
282
+ 263,
283
+ 264,
284
+ 265,
285
+ 266,
286
+ 267,
287
+ 268,
288
+ 269,
289
+ 270,
290
+ 271,
291
+ 272,
292
+ 273,
293
+ 274,
294
+ 275,
295
+ 276,
296
+ 277,
297
+ 278,
298
+ 279,
299
+ 280,
300
+ 281,
301
+ 282,
302
+ 283,
303
+ 284,
304
+ 285,
305
+ 286,
306
+ 287,
307
+ 288,
308
+ 289,
309
+ 290,
310
+ 291,
311
+ 292,
312
+ 293,
313
+ 294,
314
+ 295,
315
+ 296,
316
+ 297,
317
+ 298,
318
+ 299,
319
+ 300,
320
+ 301,
321
+ 302,
322
+ 303,
323
+ 304,
324
+ 305,
325
+ 306,
326
+ 307,
327
+ 308,
328
+ 309,
329
+ 310,
330
+ 311,
331
+ 312,
332
+ 313,
333
+ 314,
334
+ 315,
335
+ 316,
336
+ 317,
337
+ 318,
338
+ 319,
339
+ 320,
340
+ 321,
341
+ 322,
342
+ 323,
343
+ 324,
344
+ 325,
345
+ 326,
346
+ 327,
347
+ 328,
348
+ 329,
349
+ 330,
350
+ 331,
351
+ 332,
352
+ 333,
353
+ 334,
354
+ 335,
355
+ 336,
356
+ 337,
357
+ 338,
358
+ 339,
359
+ 340,
360
+ 341,
361
+ 342,
362
+ 343,
363
+ 344,
364
+ 345,
365
+ 346,
366
+ 347,
367
+ 348,
368
+ 349,
369
+ 350,
370
+ 351,
371
+ 352,
372
+ 353,
373
+ 354,
374
+ 355,
375
+ 356,
376
+ 357,
377
+ 358,
378
+ 359,
379
+ 360,
380
+ 361,
381
+ 362,
382
+ 363,
383
+ 364,
384
+ 365,
385
+ 366,
386
+ 367,
387
+ 368,
388
+ 369,
389
+ 370,
390
+ 371,
391
+ 372,
392
+ 373,
393
+ 374
394
+ ]
395
+ }
396
+ ]
checkpoints/dagger/1iter/checkpoints/018000/training_state/optimizer_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1db3b75e036ad0d4404e7230445d7b530b16067f9f9b7d9944b744e77781d775
3
+ size 2014217744
checkpoints/dagger/1iter/checkpoints/018000/training_state/rng_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0dd0be7ff45c91167174e88ff8e2c3e0cbba2a797256bdd90d8341c8a85a976
3
+ size 15708
checkpoints/dagger/1iter/checkpoints/018000/training_state/scheduler_state.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_lrs": [
3
+ 5e-05
4
+ ],
5
+ "last_epoch": 18000,
6
+ "_step_count": 18001,
7
+ "_is_initial": false,
8
+ "_get_lr_called_within_step": false,
9
+ "_last_lr": [
10
+ 9.132549638500307e-06
11
+ ],
12
+ "lr_lambdas": [
13
+ null
14
+ ]
15
+ }
checkpoints/dagger/1iter/checkpoints/018000/training_state/training_step.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "step": 18000
3
+ }
checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/config.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "diffusion",
3
+ "n_obs_steps": 2,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 6
9
+ ]
10
+ },
11
+ "observation.images.front": {
12
+ "type": "VISUAL",
13
+ "shape": [
14
+ 3,
15
+ 480,
16
+ 640
17
+ ]
18
+ }
19
+ },
20
+ "output_features": {
21
+ "action": {
22
+ "type": "ACTION",
23
+ "shape": [
24
+ 6
25
+ ]
26
+ }
27
+ },
28
+ "device": "cuda",
29
+ "use_amp": false,
30
+ "use_peft": false,
31
+ "push_to_hub": false,
32
+ "repo_id": null,
33
+ "private": null,
34
+ "tags": null,
35
+ "license": null,
36
+ "pretrained_path": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_specialized_523_dataset_base_from_dino_780_new_config_11H47/checkpoints/010000/pretrained_model",
37
+ "horizon": 16,
38
+ "n_action_steps": 8,
39
+ "normalization_mapping": {
40
+ "VISUAL": "MEAN_STD",
41
+ "STATE": "MIN_MAX",
42
+ "ACTION": "MIN_MAX"
43
+ },
44
+ "drop_n_last_frames": 7,
45
+ "vision_backbone_type": "dinov2",
46
+ "vision_encoder_name": "facebook/dinov2-small",
47
+ "freeze_vision_encoder": true,
48
+ "vision_backbone": "resnet18",
49
+ "resize_shape": [
50
+ 224,
51
+ 224
52
+ ],
53
+ "crop_ratio": 1.0,
54
+ "crop_shape": null,
55
+ "crop_is_random": true,
56
+ "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
57
+ "use_group_norm": false,
58
+ "spatial_softmax_num_keypoints": 32,
59
+ "vit_pool_type": "spatial_softmax",
60
+ "vit_feature_dim": 512,
61
+ "use_separate_rgb_encoder_per_camera": true,
62
+ "down_dims": [
63
+ 512,
64
+ 1024,
65
+ 2048
66
+ ],
67
+ "kernel_size": 5,
68
+ "n_groups": 8,
69
+ "diffusion_step_embed_dim": 128,
70
+ "use_film_scale_modulation": true,
71
+ "noise_scheduler_type": "DDPM",
72
+ "num_train_timesteps": 100,
73
+ "beta_schedule": "squaredcos_cap_v2",
74
+ "beta_start": 0.0001,
75
+ "beta_end": 0.02,
76
+ "prediction_type": "epsilon",
77
+ "clip_sample": true,
78
+ "clip_sample_range": 1.0,
79
+ "num_inference_steps": 100,
80
+ "compile_model": false,
81
+ "compile_mode": "reduce-overhead",
82
+ "do_mask_loss_for_padding": false,
83
+ "optimizer_lr": 5e-05,
84
+ "optimizer_betas": [
85
+ 0.95,
86
+ 0.999
87
+ ],
88
+ "optimizer_eps": 1e-08,
89
+ "optimizer_weight_decay": 1e-06,
90
+ "scheduler_name": "cosine",
91
+ "scheduler_warmup_steps": 100
92
+ }
checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:db7a7d9711a0a81e74e068ef0a011b108c8daf523bc68e22da623a64bf1b5ee5
3
+ size 1095367712
checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_postprocessor.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "unnormalizer_processor",
6
+ "config": {
7
+ "eps": 1e-08,
8
+ "features": {
9
+ "action": {
10
+ "type": "ACTION",
11
+ "shape": [
12
+ 6
13
+ ]
14
+ }
15
+ },
16
+ "norm_map": {
17
+ "VISUAL": "MEAN_STD",
18
+ "STATE": "MIN_MAX",
19
+ "ACTION": "MIN_MAX"
20
+ }
21
+ },
22
+ "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
23
+ },
24
+ {
25
+ "registry_name": "device_processor",
26
+ "config": {
27
+ "device": "cpu",
28
+ "float_dtype": null
29
+ }
30
+ }
31
+ ]
32
+ }
checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2cffd05b8fcb092f386f0047017f12099a576aa260612cfb4a12f36d9746bfa
3
+ size 6560
checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_preprocessor.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {}
8
+ }
9
+ },
10
+ {
11
+ "registry_name": "to_batch_processor",
12
+ "config": {}
13
+ },
14
+ {
15
+ "registry_name": "device_processor",
16
+ "config": {
17
+ "device": "cuda",
18
+ "float_dtype": null
19
+ }
20
+ },
21
+ {
22
+ "registry_name": "normalizer_processor",
23
+ "config": {
24
+ "eps": 1e-08,
25
+ "features": {
26
+ "observation.state": {
27
+ "type": "STATE",
28
+ "shape": [
29
+ 6
30
+ ]
31
+ },
32
+ "observation.images.front": {
33
+ "type": "VISUAL",
34
+ "shape": [
35
+ 3,
36
+ 480,
37
+ 640
38
+ ]
39
+ },
40
+ "action": {
41
+ "type": "ACTION",
42
+ "shape": [
43
+ 6
44
+ ]
45
+ }
46
+ },
47
+ "norm_map": {
48
+ "VISUAL": "MEAN_STD",
49
+ "STATE": "MIN_MAX",
50
+ "ACTION": "MIN_MAX"
51
+ }
52
+ },
53
+ "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors"
54
+ }
55
+ ]
56
+ }
checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2cffd05b8fcb092f386f0047017f12099a576aa260612cfb4a12f36d9746bfa
3
+ size 6560
checkpoints/dagger/1iter/checkpoints/020000/pretrained_model/train_config.json ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "local/dagger_episodes",
4
+ "root": "/home/shadeform/Desktop/robot_learning/base_new_dataset_dagger_628",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "same_cloth_color": false,
11
+ "tfs": {
12
+ "color_jitter": {
13
+ "weight": 1.5,
14
+ "type": "ColorJitter",
15
+ "kwargs": {
16
+ "brightness": [
17
+ 0.7,
18
+ 1.3
19
+ ],
20
+ "contrast": [
21
+ 0.7,
22
+ 1.3
23
+ ],
24
+ "saturation": [
25
+ 0.4,
26
+ 1.6
27
+ ],
28
+ "hue": [
29
+ -0.5,
30
+ 0.5
31
+ ]
32
+ }
33
+ },
34
+ "grayscale": {
35
+ "weight": 0.5,
36
+ "type": "RandomGrayscale",
37
+ "kwargs": {
38
+ "p": 1.0
39
+ }
40
+ },
41
+ "shadow": {
42
+ "weight": 0.8,
43
+ "type": "RandomShadow",
44
+ "kwargs": {
45
+ "num_shadows": [
46
+ 1,
47
+ 2
48
+ ],
49
+ "intensity": [
50
+ 0.4,
51
+ 0.7
52
+ ],
53
+ "blur_sigma": [
54
+ 4.0,
55
+ 20.0
56
+ ]
57
+ }
58
+ },
59
+ "highlight": {
60
+ "weight": 0.5,
61
+ "type": "RandomHighlight",
62
+ "kwargs": {
63
+ "num_highlights": [
64
+ 0,
65
+ 1
66
+ ],
67
+ "intensity": [
68
+ 0.2,
69
+ 0.5
70
+ ],
71
+ "sigma": [
72
+ 20.0,
73
+ 60.0
74
+ ]
75
+ }
76
+ },
77
+ "gamma": {
78
+ "weight": 0.7,
79
+ "type": "RandomGamma",
80
+ "kwargs": {
81
+ "gamma": [
82
+ 0.6,
83
+ 1.4
84
+ ]
85
+ }
86
+ },
87
+ "blur": {
88
+ "weight": 0.0,
89
+ "type": "GaussianBlur",
90
+ "kwargs": {
91
+ "kernel_size": [
92
+ 3,
93
+ 3
94
+ ],
95
+ "sigma": [
96
+ 0.1,
97
+ 1.5
98
+ ]
99
+ }
100
+ },
101
+ "jpeg": {
102
+ "weight": 0.0,
103
+ "type": "RandomJPEG",
104
+ "kwargs": {
105
+ "quality": [
106
+ 60,
107
+ 95
108
+ ]
109
+ }
110
+ },
111
+ "crop": {
112
+ "weight": 0.5,
113
+ "type": "RandomCropPreserveSize",
114
+ "kwargs": {
115
+ "scale": [
116
+ 0.97,
117
+ 1.0
118
+ ],
119
+ "ratio": [
120
+ 0.99,
121
+ 1.01
122
+ ]
123
+ }
124
+ },
125
+ "affine": {
126
+ "weight": 0.0,
127
+ "type": "RandomAffine",
128
+ "kwargs": {
129
+ "degrees": [
130
+ -5.0,
131
+ 5.0
132
+ ],
133
+ "translate": [
134
+ 0.05,
135
+ 0.05
136
+ ]
137
+ }
138
+ },
139
+ "sharpness": {
140
+ "weight": 1.0,
141
+ "type": "SharpnessJitter",
142
+ "kwargs": {
143
+ "sharpness": [
144
+ 0.5,
145
+ 1.5
146
+ ]
147
+ }
148
+ }
149
+ }
150
+ },
151
+ "revision": null,
152
+ "use_imagenet_stats": true,
153
+ "video_backend": "pyav",
154
+ "return_uint8": false,
155
+ "streaming": false
156
+ },
157
+ "env": null,
158
+ "policy": {
159
+ "type": "diffusion",
160
+ "n_obs_steps": 2,
161
+ "input_features": {
162
+ "observation.state": {
163
+ "type": "STATE",
164
+ "shape": [
165
+ 6
166
+ ]
167
+ },
168
+ "observation.images.front": {
169
+ "type": "VISUAL",
170
+ "shape": [
171
+ 3,
172
+ 480,
173
+ 640
174
+ ]
175
+ }
176
+ },
177
+ "output_features": {
178
+ "action": {
179
+ "type": "ACTION",
180
+ "shape": [
181
+ 6
182
+ ]
183
+ }
184
+ },
185
+ "device": "cuda",
186
+ "use_amp": false,
187
+ "use_peft": false,
188
+ "push_to_hub": false,
189
+ "repo_id": null,
190
+ "private": null,
191
+ "tags": null,
192
+ "license": null,
193
+ "pretrained_path": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_specialized_523_dataset_base_from_dino_780_new_config_11H47/checkpoints/010000/pretrained_model",
194
+ "horizon": 16,
195
+ "n_action_steps": 8,
196
+ "normalization_mapping": {
197
+ "VISUAL": "MEAN_STD",
198
+ "STATE": "MIN_MAX",
199
+ "ACTION": "MIN_MAX"
200
+ },
201
+ "drop_n_last_frames": 7,
202
+ "vision_backbone_type": "dinov2",
203
+ "vision_encoder_name": "facebook/dinov2-small",
204
+ "freeze_vision_encoder": true,
205
+ "vision_backbone": "resnet18",
206
+ "resize_shape": [
207
+ 224,
208
+ 224
209
+ ],
210
+ "crop_ratio": 1.0,
211
+ "crop_shape": null,
212
+ "crop_is_random": true,
213
+ "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
214
+ "use_group_norm": false,
215
+ "spatial_softmax_num_keypoints": 32,
216
+ "vit_pool_type": "spatial_softmax",
217
+ "vit_feature_dim": 512,
218
+ "use_separate_rgb_encoder_per_camera": true,
219
+ "down_dims": [
220
+ 512,
221
+ 1024,
222
+ 2048
223
+ ],
224
+ "kernel_size": 5,
225
+ "n_groups": 8,
226
+ "diffusion_step_embed_dim": 128,
227
+ "use_film_scale_modulation": true,
228
+ "noise_scheduler_type": "DDPM",
229
+ "num_train_timesteps": 100,
230
+ "beta_schedule": "squaredcos_cap_v2",
231
+ "beta_start": 0.0001,
232
+ "beta_end": 0.02,
233
+ "prediction_type": "epsilon",
234
+ "clip_sample": true,
235
+ "clip_sample_range": 1.0,
236
+ "num_inference_steps": 100,
237
+ "compile_model": false,
238
+ "compile_mode": "reduce-overhead",
239
+ "do_mask_loss_for_padding": false,
240
+ "optimizer_lr": 5e-05,
241
+ "optimizer_betas": [
242
+ 0.95,
243
+ 0.999
244
+ ],
245
+ "optimizer_eps": 1e-08,
246
+ "optimizer_weight_decay": 1e-06,
247
+ "scheduler_name": "cosine",
248
+ "scheduler_warmup_steps": 100
249
+ },
250
+ "reward_model": null,
251
+ "output_dir": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_dagger_iter1_25000steps_20260520_173844",
252
+ "job_name": "diffusion_dagger_iter1_25000steps_20260520_173844",
253
+ "resume": false,
254
+ "seed": 1000,
255
+ "cudnn_deterministic": false,
256
+ "num_workers": 16,
257
+ "batch_size": 64,
258
+ "prefetch_factor": 4,
259
+ "persistent_workers": true,
260
+ "steps": 25000,
261
+ "eval_freq": 0,
262
+ "log_freq": 20,
263
+ "tolerance_s": 0.0001,
264
+ "save_checkpoint": true,
265
+ "save_freq": 2000,
266
+ "use_policy_training_preset": true,
267
+ "optimizer": {
268
+ "type": "adam",
269
+ "lr": 5e-05,
270
+ "weight_decay": 1e-06,
271
+ "grad_clip_norm": 10.0,
272
+ "betas": [
273
+ 0.95,
274
+ 0.999
275
+ ],
276
+ "eps": 1e-08
277
+ },
278
+ "scheduler": {
279
+ "type": "diffuser",
280
+ "num_warmup_steps": 100,
281
+ "name": "cosine"
282
+ },
283
+ "eval": {
284
+ "n_episodes": 50,
285
+ "batch_size": 19,
286
+ "use_async_envs": true
287
+ },
288
+ "wandb": {
289
+ "enable": true,
290
+ "disable_artifact": false,
291
+ "project": "lerobot",
292
+ "entity": null,
293
+ "notes": null,
294
+ "run_id": "ql0pvhal",
295
+ "mode": null,
296
+ "add_tags": true
297
+ },
298
+ "peft": null,
299
+ "sample_weighting": null,
300
+ "rename_map": {},
301
+ "checkpoint_path": null
302
+ }
checkpoints/dagger/1iter/checkpoints/020000/training_state/optimizer_param_groups.json ADDED
@@ -0,0 +1,396 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "lr": 4.811719449200755e-06,
4
+ "betas": [
5
+ 0.95,
6
+ 0.999
7
+ ],
8
+ "eps": 1e-08,
9
+ "weight_decay": 1e-06,
10
+ "amsgrad": false,
11
+ "maximize": false,
12
+ "foreach": null,
13
+ "capturable": false,
14
+ "differentiable": false,
15
+ "fused": null,
16
+ "decoupled_weight_decay": false,
17
+ "initial_lr": 5e-05,
18
+ "params": [
19
+ 0,
20
+ 1,
21
+ 2,
22
+ 3,
23
+ 4,
24
+ 5,
25
+ 6,
26
+ 7,
27
+ 8,
28
+ 9,
29
+ 10,
30
+ 11,
31
+ 12,
32
+ 13,
33
+ 14,
34
+ 15,
35
+ 16,
36
+ 17,
37
+ 18,
38
+ 19,
39
+ 20,
40
+ 21,
41
+ 22,
42
+ 23,
43
+ 24,
44
+ 25,
45
+ 26,
46
+ 27,
47
+ 28,
48
+ 29,
49
+ 30,
50
+ 31,
51
+ 32,
52
+ 33,
53
+ 34,
54
+ 35,
55
+ 36,
56
+ 37,
57
+ 38,
58
+ 39,
59
+ 40,
60
+ 41,
61
+ 42,
62
+ 43,
63
+ 44,
64
+ 45,
65
+ 46,
66
+ 47,
67
+ 48,
68
+ 49,
69
+ 50,
70
+ 51,
71
+ 52,
72
+ 53,
73
+ 54,
74
+ 55,
75
+ 56,
76
+ 57,
77
+ 58,
78
+ 59,
79
+ 60,
80
+ 61,
81
+ 62,
82
+ 63,
83
+ 64,
84
+ 65,
85
+ 66,
86
+ 67,
87
+ 68,
88
+ 69,
89
+ 70,
90
+ 71,
91
+ 72,
92
+ 73,
93
+ 74,
94
+ 75,
95
+ 76,
96
+ 77,
97
+ 78,
98
+ 79,
99
+ 80,
100
+ 81,
101
+ 82,
102
+ 83,
103
+ 84,
104
+ 85,
105
+ 86,
106
+ 87,
107
+ 88,
108
+ 89,
109
+ 90,
110
+ 91,
111
+ 92,
112
+ 93,
113
+ 94,
114
+ 95,
115
+ 96,
116
+ 97,
117
+ 98,
118
+ 99,
119
+ 100,
120
+ 101,
121
+ 102,
122
+ 103,
123
+ 104,
124
+ 105,
125
+ 106,
126
+ 107,
127
+ 108,
128
+ 109,
129
+ 110,
130
+ 111,
131
+ 112,
132
+ 113,
133
+ 114,
134
+ 115,
135
+ 116,
136
+ 117,
137
+ 118,
138
+ 119,
139
+ 120,
140
+ 121,
141
+ 122,
142
+ 123,
143
+ 124,
144
+ 125,
145
+ 126,
146
+ 127,
147
+ 128,
148
+ 129,
149
+ 130,
150
+ 131,
151
+ 132,
152
+ 133,
153
+ 134,
154
+ 135,
155
+ 136,
156
+ 137,
157
+ 138,
158
+ 139,
159
+ 140,
160
+ 141,
161
+ 142,
162
+ 143,
163
+ 144,
164
+ 145,
165
+ 146,
166
+ 147,
167
+ 148,
168
+ 149,
169
+ 150,
170
+ 151,
171
+ 152,
172
+ 153,
173
+ 154,
174
+ 155,
175
+ 156,
176
+ 157,
177
+ 158,
178
+ 159,
179
+ 160,
180
+ 161,
181
+ 162,
182
+ 163,
183
+ 164,
184
+ 165,
185
+ 166,
186
+ 167,
187
+ 168,
188
+ 169,
189
+ 170,
190
+ 171,
191
+ 172,
192
+ 173,
193
+ 174,
194
+ 175,
195
+ 176,
196
+ 177,
197
+ 178,
198
+ 179,
199
+ 180,
200
+ 181,
201
+ 182,
202
+ 183,
203
+ 184,
204
+ 185,
205
+ 186,
206
+ 187,
207
+ 188,
208
+ 189,
209
+ 190,
210
+ 191,
211
+ 192,
212
+ 193,
213
+ 194,
214
+ 195,
215
+ 196,
216
+ 197,
217
+ 198,
218
+ 199,
219
+ 200,
220
+ 201,
221
+ 202,
222
+ 203,
223
+ 204,
224
+ 205,
225
+ 206,
226
+ 207,
227
+ 208,
228
+ 209,
229
+ 210,
230
+ 211,
231
+ 212,
232
+ 213,
233
+ 214,
234
+ 215,
235
+ 216,
236
+ 217,
237
+ 218,
238
+ 219,
239
+ 220,
240
+ 221,
241
+ 222,
242
+ 223,
243
+ 224,
244
+ 225,
245
+ 226,
246
+ 227,
247
+ 228,
248
+ 229,
249
+ 230,
250
+ 231,
251
+ 232,
252
+ 233,
253
+ 234,
254
+ 235,
255
+ 236,
256
+ 237,
257
+ 238,
258
+ 239,
259
+ 240,
260
+ 241,
261
+ 242,
262
+ 243,
263
+ 244,
264
+ 245,
265
+ 246,
266
+ 247,
267
+ 248,
268
+ 249,
269
+ 250,
270
+ 251,
271
+ 252,
272
+ 253,
273
+ 254,
274
+ 255,
275
+ 256,
276
+ 257,
277
+ 258,
278
+ 259,
279
+ 260,
280
+ 261,
281
+ 262,
282
+ 263,
283
+ 264,
284
+ 265,
285
+ 266,
286
+ 267,
287
+ 268,
288
+ 269,
289
+ 270,
290
+ 271,
291
+ 272,
292
+ 273,
293
+ 274,
294
+ 275,
295
+ 276,
296
+ 277,
297
+ 278,
298
+ 279,
299
+ 280,
300
+ 281,
301
+ 282,
302
+ 283,
303
+ 284,
304
+ 285,
305
+ 286,
306
+ 287,
307
+ 288,
308
+ 289,
309
+ 290,
310
+ 291,
311
+ 292,
312
+ 293,
313
+ 294,
314
+ 295,
315
+ 296,
316
+ 297,
317
+ 298,
318
+ 299,
319
+ 300,
320
+ 301,
321
+ 302,
322
+ 303,
323
+ 304,
324
+ 305,
325
+ 306,
326
+ 307,
327
+ 308,
328
+ 309,
329
+ 310,
330
+ 311,
331
+ 312,
332
+ 313,
333
+ 314,
334
+ 315,
335
+ 316,
336
+ 317,
337
+ 318,
338
+ 319,
339
+ 320,
340
+ 321,
341
+ 322,
342
+ 323,
343
+ 324,
344
+ 325,
345
+ 326,
346
+ 327,
347
+ 328,
348
+ 329,
349
+ 330,
350
+ 331,
351
+ 332,
352
+ 333,
353
+ 334,
354
+ 335,
355
+ 336,
356
+ 337,
357
+ 338,
358
+ 339,
359
+ 340,
360
+ 341,
361
+ 342,
362
+ 343,
363
+ 344,
364
+ 345,
365
+ 346,
366
+ 347,
367
+ 348,
368
+ 349,
369
+ 350,
370
+ 351,
371
+ 352,
372
+ 353,
373
+ 354,
374
+ 355,
375
+ 356,
376
+ 357,
377
+ 358,
378
+ 359,
379
+ 360,
380
+ 361,
381
+ 362,
382
+ 363,
383
+ 364,
384
+ 365,
385
+ 366,
386
+ 367,
387
+ 368,
388
+ 369,
389
+ 370,
390
+ 371,
391
+ 372,
392
+ 373,
393
+ 374
394
+ ]
395
+ }
396
+ ]
checkpoints/dagger/1iter/checkpoints/020000/training_state/optimizer_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f410284da5a84e8ca3d1536f30e088c45588ff6256f0f30d30172781be18cc2
3
+ size 2014217744
checkpoints/dagger/1iter/checkpoints/020000/training_state/rng_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29447813a78c938682aa8eb7345f6c9bd4d8f39298176622c7e1c62552ebc6c3
3
+ size 15708
checkpoints/dagger/1iter/checkpoints/020000/training_state/scheduler_state.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_lrs": [
3
+ 5e-05
4
+ ],
5
+ "last_epoch": 20000,
6
+ "_step_count": 20001,
7
+ "_is_initial": false,
8
+ "_get_lr_called_within_step": false,
9
+ "_last_lr": [
10
+ 4.811719449200755e-06
11
+ ],
12
+ "lr_lambdas": [
13
+ null
14
+ ]
15
+ }
checkpoints/dagger/1iter/checkpoints/020000/training_state/training_step.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "step": 20000
3
+ }
checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/config.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "diffusion",
3
+ "n_obs_steps": 2,
4
+ "input_features": {
5
+ "observation.state": {
6
+ "type": "STATE",
7
+ "shape": [
8
+ 6
9
+ ]
10
+ },
11
+ "observation.images.front": {
12
+ "type": "VISUAL",
13
+ "shape": [
14
+ 3,
15
+ 480,
16
+ 640
17
+ ]
18
+ }
19
+ },
20
+ "output_features": {
21
+ "action": {
22
+ "type": "ACTION",
23
+ "shape": [
24
+ 6
25
+ ]
26
+ }
27
+ },
28
+ "device": "cuda",
29
+ "use_amp": false,
30
+ "use_peft": false,
31
+ "push_to_hub": false,
32
+ "repo_id": null,
33
+ "private": null,
34
+ "tags": null,
35
+ "license": null,
36
+ "pretrained_path": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_specialized_523_dataset_base_from_dino_780_new_config_11H47/checkpoints/010000/pretrained_model",
37
+ "horizon": 16,
38
+ "n_action_steps": 8,
39
+ "normalization_mapping": {
40
+ "VISUAL": "MEAN_STD",
41
+ "STATE": "MIN_MAX",
42
+ "ACTION": "MIN_MAX"
43
+ },
44
+ "drop_n_last_frames": 7,
45
+ "vision_backbone_type": "dinov2",
46
+ "vision_encoder_name": "facebook/dinov2-small",
47
+ "freeze_vision_encoder": true,
48
+ "vision_backbone": "resnet18",
49
+ "resize_shape": [
50
+ 224,
51
+ 224
52
+ ],
53
+ "crop_ratio": 1.0,
54
+ "crop_shape": null,
55
+ "crop_is_random": true,
56
+ "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
57
+ "use_group_norm": false,
58
+ "spatial_softmax_num_keypoints": 32,
59
+ "vit_pool_type": "spatial_softmax",
60
+ "vit_feature_dim": 512,
61
+ "use_separate_rgb_encoder_per_camera": true,
62
+ "down_dims": [
63
+ 512,
64
+ 1024,
65
+ 2048
66
+ ],
67
+ "kernel_size": 5,
68
+ "n_groups": 8,
69
+ "diffusion_step_embed_dim": 128,
70
+ "use_film_scale_modulation": true,
71
+ "noise_scheduler_type": "DDPM",
72
+ "num_train_timesteps": 100,
73
+ "beta_schedule": "squaredcos_cap_v2",
74
+ "beta_start": 0.0001,
75
+ "beta_end": 0.02,
76
+ "prediction_type": "epsilon",
77
+ "clip_sample": true,
78
+ "clip_sample_range": 1.0,
79
+ "num_inference_steps": 100,
80
+ "compile_model": false,
81
+ "compile_mode": "reduce-overhead",
82
+ "do_mask_loss_for_padding": false,
83
+ "optimizer_lr": 5e-05,
84
+ "optimizer_betas": [
85
+ 0.95,
86
+ 0.999
87
+ ],
88
+ "optimizer_eps": 1e-08,
89
+ "optimizer_weight_decay": 1e-06,
90
+ "scheduler_name": "cosine",
91
+ "scheduler_warmup_steps": 100
92
+ }
checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7124a4f4ae43d00c3d13f32e1c4d779acdcda2de064770391e58fd55fe76937
3
+ size 1095367712
checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_postprocessor.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_postprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "unnormalizer_processor",
6
+ "config": {
7
+ "eps": 1e-08,
8
+ "features": {
9
+ "action": {
10
+ "type": "ACTION",
11
+ "shape": [
12
+ 6
13
+ ]
14
+ }
15
+ },
16
+ "norm_map": {
17
+ "VISUAL": "MEAN_STD",
18
+ "STATE": "MIN_MAX",
19
+ "ACTION": "MIN_MAX"
20
+ }
21
+ },
22
+ "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
23
+ },
24
+ {
25
+ "registry_name": "device_processor",
26
+ "config": {
27
+ "device": "cpu",
28
+ "float_dtype": null
29
+ }
30
+ }
31
+ ]
32
+ }
checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2cffd05b8fcb092f386f0047017f12099a576aa260612cfb4a12f36d9746bfa
3
+ size 6560
checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_preprocessor.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "policy_preprocessor",
3
+ "steps": [
4
+ {
5
+ "registry_name": "rename_observations_processor",
6
+ "config": {
7
+ "rename_map": {}
8
+ }
9
+ },
10
+ {
11
+ "registry_name": "to_batch_processor",
12
+ "config": {}
13
+ },
14
+ {
15
+ "registry_name": "device_processor",
16
+ "config": {
17
+ "device": "cuda",
18
+ "float_dtype": null
19
+ }
20
+ },
21
+ {
22
+ "registry_name": "normalizer_processor",
23
+ "config": {
24
+ "eps": 1e-08,
25
+ "features": {
26
+ "observation.state": {
27
+ "type": "STATE",
28
+ "shape": [
29
+ 6
30
+ ]
31
+ },
32
+ "observation.images.front": {
33
+ "type": "VISUAL",
34
+ "shape": [
35
+ 3,
36
+ 480,
37
+ 640
38
+ ]
39
+ },
40
+ "action": {
41
+ "type": "ACTION",
42
+ "shape": [
43
+ 6
44
+ ]
45
+ }
46
+ },
47
+ "norm_map": {
48
+ "VISUAL": "MEAN_STD",
49
+ "STATE": "MIN_MAX",
50
+ "ACTION": "MIN_MAX"
51
+ }
52
+ },
53
+ "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors"
54
+ }
55
+ ]
56
+ }
checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2cffd05b8fcb092f386f0047017f12099a576aa260612cfb4a12f36d9746bfa
3
+ size 6560
checkpoints/dagger/1iter/checkpoints/022000/pretrained_model/train_config.json ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "repo_id": "local/dagger_episodes",
4
+ "root": "/home/shadeform/Desktop/robot_learning/base_new_dataset_dagger_628",
5
+ "episodes": null,
6
+ "image_transforms": {
7
+ "enable": false,
8
+ "max_num_transforms": 3,
9
+ "random_order": false,
10
+ "same_cloth_color": false,
11
+ "tfs": {
12
+ "color_jitter": {
13
+ "weight": 1.5,
14
+ "type": "ColorJitter",
15
+ "kwargs": {
16
+ "brightness": [
17
+ 0.7,
18
+ 1.3
19
+ ],
20
+ "contrast": [
21
+ 0.7,
22
+ 1.3
23
+ ],
24
+ "saturation": [
25
+ 0.4,
26
+ 1.6
27
+ ],
28
+ "hue": [
29
+ -0.5,
30
+ 0.5
31
+ ]
32
+ }
33
+ },
34
+ "grayscale": {
35
+ "weight": 0.5,
36
+ "type": "RandomGrayscale",
37
+ "kwargs": {
38
+ "p": 1.0
39
+ }
40
+ },
41
+ "shadow": {
42
+ "weight": 0.8,
43
+ "type": "RandomShadow",
44
+ "kwargs": {
45
+ "num_shadows": [
46
+ 1,
47
+ 2
48
+ ],
49
+ "intensity": [
50
+ 0.4,
51
+ 0.7
52
+ ],
53
+ "blur_sigma": [
54
+ 4.0,
55
+ 20.0
56
+ ]
57
+ }
58
+ },
59
+ "highlight": {
60
+ "weight": 0.5,
61
+ "type": "RandomHighlight",
62
+ "kwargs": {
63
+ "num_highlights": [
64
+ 0,
65
+ 1
66
+ ],
67
+ "intensity": [
68
+ 0.2,
69
+ 0.5
70
+ ],
71
+ "sigma": [
72
+ 20.0,
73
+ 60.0
74
+ ]
75
+ }
76
+ },
77
+ "gamma": {
78
+ "weight": 0.7,
79
+ "type": "RandomGamma",
80
+ "kwargs": {
81
+ "gamma": [
82
+ 0.6,
83
+ 1.4
84
+ ]
85
+ }
86
+ },
87
+ "blur": {
88
+ "weight": 0.0,
89
+ "type": "GaussianBlur",
90
+ "kwargs": {
91
+ "kernel_size": [
92
+ 3,
93
+ 3
94
+ ],
95
+ "sigma": [
96
+ 0.1,
97
+ 1.5
98
+ ]
99
+ }
100
+ },
101
+ "jpeg": {
102
+ "weight": 0.0,
103
+ "type": "RandomJPEG",
104
+ "kwargs": {
105
+ "quality": [
106
+ 60,
107
+ 95
108
+ ]
109
+ }
110
+ },
111
+ "crop": {
112
+ "weight": 0.5,
113
+ "type": "RandomCropPreserveSize",
114
+ "kwargs": {
115
+ "scale": [
116
+ 0.97,
117
+ 1.0
118
+ ],
119
+ "ratio": [
120
+ 0.99,
121
+ 1.01
122
+ ]
123
+ }
124
+ },
125
+ "affine": {
126
+ "weight": 0.0,
127
+ "type": "RandomAffine",
128
+ "kwargs": {
129
+ "degrees": [
130
+ -5.0,
131
+ 5.0
132
+ ],
133
+ "translate": [
134
+ 0.05,
135
+ 0.05
136
+ ]
137
+ }
138
+ },
139
+ "sharpness": {
140
+ "weight": 1.0,
141
+ "type": "SharpnessJitter",
142
+ "kwargs": {
143
+ "sharpness": [
144
+ 0.5,
145
+ 1.5
146
+ ]
147
+ }
148
+ }
149
+ }
150
+ },
151
+ "revision": null,
152
+ "use_imagenet_stats": true,
153
+ "video_backend": "pyav",
154
+ "return_uint8": false,
155
+ "streaming": false
156
+ },
157
+ "env": null,
158
+ "policy": {
159
+ "type": "diffusion",
160
+ "n_obs_steps": 2,
161
+ "input_features": {
162
+ "observation.state": {
163
+ "type": "STATE",
164
+ "shape": [
165
+ 6
166
+ ]
167
+ },
168
+ "observation.images.front": {
169
+ "type": "VISUAL",
170
+ "shape": [
171
+ 3,
172
+ 480,
173
+ 640
174
+ ]
175
+ }
176
+ },
177
+ "output_features": {
178
+ "action": {
179
+ "type": "ACTION",
180
+ "shape": [
181
+ 6
182
+ ]
183
+ }
184
+ },
185
+ "device": "cuda",
186
+ "use_amp": false,
187
+ "use_peft": false,
188
+ "push_to_hub": false,
189
+ "repo_id": null,
190
+ "private": null,
191
+ "tags": null,
192
+ "license": null,
193
+ "pretrained_path": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_specialized_523_dataset_base_from_dino_780_new_config_11H47/checkpoints/010000/pretrained_model",
194
+ "horizon": 16,
195
+ "n_action_steps": 8,
196
+ "normalization_mapping": {
197
+ "VISUAL": "MEAN_STD",
198
+ "STATE": "MIN_MAX",
199
+ "ACTION": "MIN_MAX"
200
+ },
201
+ "drop_n_last_frames": 7,
202
+ "vision_backbone_type": "dinov2",
203
+ "vision_encoder_name": "facebook/dinov2-small",
204
+ "freeze_vision_encoder": true,
205
+ "vision_backbone": "resnet18",
206
+ "resize_shape": [
207
+ 224,
208
+ 224
209
+ ],
210
+ "crop_ratio": 1.0,
211
+ "crop_shape": null,
212
+ "crop_is_random": true,
213
+ "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
214
+ "use_group_norm": false,
215
+ "spatial_softmax_num_keypoints": 32,
216
+ "vit_pool_type": "spatial_softmax",
217
+ "vit_feature_dim": 512,
218
+ "use_separate_rgb_encoder_per_camera": true,
219
+ "down_dims": [
220
+ 512,
221
+ 1024,
222
+ 2048
223
+ ],
224
+ "kernel_size": 5,
225
+ "n_groups": 8,
226
+ "diffusion_step_embed_dim": 128,
227
+ "use_film_scale_modulation": true,
228
+ "noise_scheduler_type": "DDPM",
229
+ "num_train_timesteps": 100,
230
+ "beta_schedule": "squaredcos_cap_v2",
231
+ "beta_start": 0.0001,
232
+ "beta_end": 0.02,
233
+ "prediction_type": "epsilon",
234
+ "clip_sample": true,
235
+ "clip_sample_range": 1.0,
236
+ "num_inference_steps": 100,
237
+ "compile_model": false,
238
+ "compile_mode": "reduce-overhead",
239
+ "do_mask_loss_for_padding": false,
240
+ "optimizer_lr": 5e-05,
241
+ "optimizer_betas": [
242
+ 0.95,
243
+ 0.999
244
+ ],
245
+ "optimizer_eps": 1e-08,
246
+ "optimizer_weight_decay": 1e-06,
247
+ "scheduler_name": "cosine",
248
+ "scheduler_warmup_steps": 100
249
+ },
250
+ "reward_model": null,
251
+ "output_dir": "/home/shadeform/Desktop/robot_learning/outputs/train/diffusion_dagger_iter1_25000steps_20260520_173844",
252
+ "job_name": "diffusion_dagger_iter1_25000steps_20260520_173844",
253
+ "resume": false,
254
+ "seed": 1000,
255
+ "cudnn_deterministic": false,
256
+ "num_workers": 16,
257
+ "batch_size": 64,
258
+ "prefetch_factor": 4,
259
+ "persistent_workers": true,
260
+ "steps": 25000,
261
+ "eval_freq": 0,
262
+ "log_freq": 20,
263
+ "tolerance_s": 0.0001,
264
+ "save_checkpoint": true,
265
+ "save_freq": 2000,
266
+ "use_policy_training_preset": true,
267
+ "optimizer": {
268
+ "type": "adam",
269
+ "lr": 5e-05,
270
+ "weight_decay": 1e-06,
271
+ "grad_clip_norm": 10.0,
272
+ "betas": [
273
+ 0.95,
274
+ 0.999
275
+ ],
276
+ "eps": 1e-08
277
+ },
278
+ "scheduler": {
279
+ "type": "diffuser",
280
+ "num_warmup_steps": 100,
281
+ "name": "cosine"
282
+ },
283
+ "eval": {
284
+ "n_episodes": 50,
285
+ "batch_size": 19,
286
+ "use_async_envs": true
287
+ },
288
+ "wandb": {
289
+ "enable": true,
290
+ "disable_artifact": false,
291
+ "project": "lerobot",
292
+ "entity": null,
293
+ "notes": null,
294
+ "run_id": "ql0pvhal",
295
+ "mode": null,
296
+ "add_tags": true
297
+ },
298
+ "peft": null,
299
+ "sample_weighting": null,
300
+ "rename_map": {},
301
+ "checkpoint_path": null
302
+ }
checkpoints/dagger/1iter/checkpoints/022000/training_state/optimizer_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cdea8bb22537fd2244886b1bbae2784f7fa7f3251f744c5d26c67431b8beea23
3
+ size 1576886272
checkpoints/dagger/1iter/checkpoints/022000/training_state/rng_state.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1540a1b158d3910f81f05f1500ed136179d9534484144d5aedcf26d7d065df04
3
+ size 15708
checkpoints/dagger/1iter/checkpoints/022000/training_state/training_step.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "step": 22000
3
+ }
checkpoints/dagger/1iter/wandb/debug-internal.log CHANGED
@@ -245,3 +245,82 @@
245
  {"time":"2026-05-20T18:08:24.975880521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
246
  {"time":"2026-05-20T18:08:39.716535976Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":840,"history_lines":8,"events_offset":235,"events_lines":2,"console_offset":865,"console_lines":9}
247
  {"time":"2026-05-20T18:08:39.877448628Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  {"time":"2026-05-20T18:08:24.975880521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
246
  {"time":"2026-05-20T18:08:39.716535976Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":840,"history_lines":8,"events_offset":235,"events_lines":2,"console_offset":865,"console_lines":9}
247
  {"time":"2026-05-20T18:08:39.877448628Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
248
+ {"time":"2026-05-20T18:08:54.716533796Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":848,"history_lines":7,"events_offset":237,"events_lines":2,"console_offset":873,"console_lines":8}
249
+ {"time":"2026-05-20T18:08:54.871911901Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
250
+ {"time":"2026-05-20T18:09:09.717065112Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":855,"history_lines":8,"events_offset":239,"events_lines":2,"console_offset":880,"console_lines":9}
251
+ {"time":"2026-05-20T18:09:09.842775704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
252
+ {"time":"2026-05-20T18:09:24.716992954Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":863,"history_lines":7,"events_offset":241,"events_lines":2,"console_offset":888,"console_lines":8}
253
+ {"time":"2026-05-20T18:09:24.905276893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
254
+ {"time":"2026-05-20T18:09:39.716505143Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":870,"history_lines":8,"events_offset":243,"events_lines":2,"console_offset":895,"console_lines":9}
255
+ {"time":"2026-05-20T18:09:39.927378967Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
256
+ {"time":"2026-05-20T18:09:54.719405323Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":878,"history_lines":7,"events_offset":245,"events_lines":2,"console_offset":903,"console_lines":8}
257
+ {"time":"2026-05-20T18:09:54.962240271Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
258
+ {"time":"2026-05-20T18:10:09.716837911Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":885,"history_lines":8,"events_offset":247,"events_lines":2,"console_offset":910,"console_lines":9}
259
+ {"time":"2026-05-20T18:10:09.923036222Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
260
+ {"time":"2026-05-20T18:10:24.717098427Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":893,"history_lines":6,"events_offset":249,"events_lines":2,"console_offset":918,"console_lines":7}
261
+ {"time":"2026-05-20T18:10:24.849016548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
262
+ {"time":"2026-05-20T18:10:39.716917501Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":899,"history_lines":5,"events_offset":251,"events_lines":2,"console_offset":925,"console_lines":6}
263
+ {"time":"2026-05-20T18:10:39.871365988Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
264
+ {"time":"2026-05-20T18:10:54.717358924Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":904,"history_lines":7,"events_offset":253,"events_lines":2,"console_offset":930,"console_lines":8}
265
+ {"time":"2026-05-20T18:10:54.966989707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
266
+ {"time":"2026-05-20T18:11:09.71665628Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":911,"history_lines":8,"events_offset":255,"events_lines":2,"console_offset":937,"console_lines":9}
267
+ {"time":"2026-05-20T18:11:09.918838619Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
268
+ {"time":"2026-05-20T18:11:24.719326062Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":919,"history_lines":8,"events_offset":257,"events_lines":2,"console_offset":945,"console_lines":9}
269
+ {"time":"2026-05-20T18:11:24.939013016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
270
+ {"time":"2026-05-20T18:11:39.717353513Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":927,"history_lines":7,"events_offset":259,"events_lines":2,"console_offset":953,"console_lines":8}
271
+ {"time":"2026-05-20T18:11:39.947273451Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
272
+ {"time":"2026-05-20T18:11:54.717226684Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":934,"history_lines":8,"events_offset":261,"events_lines":2,"console_offset":960,"console_lines":9}
273
+ {"time":"2026-05-20T18:11:54.9660831Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
274
+ {"time":"2026-05-20T18:12:09.716460033Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":942,"history_lines":7,"events_offset":263,"events_lines":2,"console_offset":968,"console_lines":8}
275
+ {"time":"2026-05-20T18:12:09.928958612Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
276
+ {"time":"2026-05-20T18:12:24.716408262Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":949,"history_lines":8,"events_offset":265,"events_lines":2,"console_offset":975,"console_lines":9}
277
+ {"time":"2026-05-20T18:12:24.857647396Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
278
+ {"time":"2026-05-20T18:12:39.717038918Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":957,"history_lines":7,"events_offset":267,"events_lines":2,"console_offset":983,"console_lines":8}
279
+ {"time":"2026-05-20T18:12:39.854732599Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
280
+ {"time":"2026-05-20T18:12:54.717825093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":964,"history_lines":8,"events_offset":269,"events_lines":2,"console_offset":990,"console_lines":9}
281
+ {"time":"2026-05-20T18:12:54.834004867Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
282
+ {"time":"2026-05-20T18:13:09.716555396Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":972,"history_lines":8,"events_offset":271,"events_lines":2,"console_offset":998,"console_lines":8}
283
+ {"time":"2026-05-20T18:13:09.8229178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
284
+ {"time":"2026-05-20T18:13:24.71717241Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":980,"history_lines":7,"events_offset":273,"events_lines":2,"console_offset":1006,"console_lines":8}
285
+ {"time":"2026-05-20T18:13:24.853483771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
286
+ {"time":"2026-05-20T18:13:39.716643723Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":987,"history_lines":8,"events_offset":275,"events_lines":2,"console_offset":1013,"console_lines":9}
287
+ {"time":"2026-05-20T18:13:39.868988756Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
288
+ {"time":"2026-05-20T18:13:54.716391483Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":995,"history_lines":4,"events_offset":277,"events_lines":2,"console_offset":1021,"console_lines":5}
289
+ {"time":"2026-05-20T18:13:54.833916452Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
290
+ {"time":"2026-05-20T18:14:09.718337714Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":999,"history_lines":6,"events_offset":279,"events_lines":2,"console_offset":1026,"console_lines":7}
291
+ {"time":"2026-05-20T18:14:09.886639112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
292
+ {"time":"2026-05-20T18:14:24.717072023Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1005,"history_lines":8,"events_offset":281,"events_lines":2,"console_offset":1032,"console_lines":9}
293
+ {"time":"2026-05-20T18:14:24.8770512Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
294
+ {"time":"2026-05-20T18:14:39.716644516Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1013,"history_lines":7,"events_offset":283,"events_lines":2,"console_offset":1040,"console_lines":8}
295
+ {"time":"2026-05-20T18:14:39.855086461Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
296
+ {"time":"2026-05-20T18:14:54.716816662Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1020,"history_lines":8,"events_offset":285,"events_lines":2,"console_offset":1047,"console_lines":9}
297
+ {"time":"2026-05-20T18:14:54.854717465Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
298
+ {"time":"2026-05-20T18:15:09.717130685Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1028,"history_lines":6,"events_offset":287,"events_lines":2,"console_offset":1055,"console_lines":7}
299
+ {"time":"2026-05-20T18:15:10.172699697Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
300
+ {"time":"2026-05-20T18:15:24.716991906Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1034,"history_lines":8,"events_offset":289,"events_lines":2,"console_offset":1061,"console_lines":8}
301
+ {"time":"2026-05-20T18:15:24.882878011Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
302
+ {"time":"2026-05-20T18:15:39.716791799Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1042,"history_lines":7,"events_offset":291,"events_lines":2,"console_offset":1069,"console_lines":8}
303
+ {"time":"2026-05-20T18:15:39.870856728Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
304
+ {"time":"2026-05-20T18:15:54.716408817Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1049,"history_lines":8,"events_offset":293,"events_lines":2,"console_offset":1076,"console_lines":8}
305
+ {"time":"2026-05-20T18:15:55.084622379Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
306
+ {"time":"2026-05-20T18:16:09.71677926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1057,"history_lines":7,"events_offset":295,"events_lines":2,"console_offset":1084,"console_lines":8}
307
+ {"time":"2026-05-20T18:16:09.868219122Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
308
+ {"time":"2026-05-20T18:16:24.716458796Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1064,"history_lines":8,"events_offset":297,"events_lines":2,"console_offset":1091,"console_lines":8}
309
+ {"time":"2026-05-20T18:16:24.963282586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
310
+ {"time":"2026-05-20T18:16:39.716595622Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1072,"history_lines":7,"events_offset":299,"events_lines":2,"console_offset":1099,"console_lines":8}
311
+ {"time":"2026-05-20T18:16:39.984253185Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
312
+ {"time":"2026-05-20T18:16:54.717499092Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1079,"history_lines":8,"events_offset":301,"events_lines":2,"console_offset":1106,"console_lines":9}
313
+ {"time":"2026-05-20T18:16:54.870864329Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
314
+ {"time":"2026-05-20T18:17:09.717171832Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1087,"history_lines":7,"events_offset":303,"events_lines":2,"console_offset":1114,"console_lines":8}
315
+ {"time":"2026-05-20T18:17:09.886232883Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
316
+ {"time":"2026-05-20T18:17:24.717170024Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1094,"history_lines":5,"events_offset":305,"events_lines":2,"console_offset":1121,"console_lines":6}
317
+ {"time":"2026-05-20T18:17:24.878279001Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
318
+ {"time":"2026-05-20T18:17:29.717154442Z","level":"ERROR","msg":"failed to write to chunk: failed to append: write /home/shadeform/Desktop/robot_learning/outputs/train/diffusion_dagger_iter1_25000steps_20260520_173844/wandb/run-20260520_173853-ql0pvhal/files/output.log: no space left on device"}
319
+ {"time":"2026-05-20T18:17:39.717020224Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":307,"events_lines":2,"console_offset":1127,"console_lines":22}
320
+ {"time":"2026-05-20T18:17:39.907862923Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
321
+ {"time":"2026-05-20T18:17:54.716618774Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":309,"events_lines":2}
322
+ {"time":"2026-05-20T18:17:54.949353147Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
323
+ {"time":"2026-05-20T18:18:09.717108368Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":311,"events_lines":2}
324
+ {"time":"2026-05-20T18:18:09.928981618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
325
+ {"time":"2026-05-20T18:18:24.716656857Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":313,"events_lines":2}
326
+ {"time":"2026-05-20T18:18:24.856513972Z","level":"INFO","msg":"filestream: request sent","status":
checkpoints/dagger/1iter/wandb/debug.log CHANGED
@@ -17,3 +17,18 @@ config: {'dataset': {'repo_id': 'local/dagger_episodes', 'root': '/home/shadefor
17
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2428] Wrapping output streams.
18
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2451] Redirects installed.
19
  2026-05-20 17:38:54,714 INFO MainThread:1855373 [wandb_init.py:init():1095] run started, returning control to user process
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2428] Wrapping output streams.
18
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2451] Redirects installed.
19
  2026-05-20 17:38:54,714 INFO MainThread:1855373 [wandb_init.py:init():1095] run started, returning control to user process
20
+ 2026-05-20 18:18:46,787 INFO wandb-AsyncioManager-main:1855373 [service_client.py:_forward_responses():122] Reached EOF.
21
+ 2026-05-20 18:18:46,787 INFO wandb-AsyncioManager-main:1855373 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles.
22
+ 2026-05-20 18:18:47,226 ERROR wandb-AsyncioManager-main:1855373 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
23
+ Traceback (most recent call last):
24
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
25
+ await fn()
26
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 105, in loop
27
+ await self._poll_and_print(timeout=None)
28
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 133, in _poll_and_print
29
+ handle = await self._interface.deliver_async(request)
30
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
31
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
32
+ handle = await self._client.deliver(request)
33
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
34
+ File "/home
checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/files/config.yaml ADDED
File without changes
checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/files/output.log CHANGED
@@ -871,4 +871,268 @@ Training: 68%|█████████████████████
871
  Training: 68%|█████████████████████████▋ | 16940/25000 [29:37<13:13, 10.16step/s]INFO 2026-05-20 18:08:34 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.05 loss:0.008 grdn:0.197 lr:1.2e-05 updt_s:0.094 data_s:0.006
872
  Training: 68%|█████████████████████████▊ | 16960/25000 [29:40<14:37, 9.16step/s]INFO 2026-05-20 18:08:36 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.05 loss:0.008 grdn:0.186 lr:1.2e-05 updt_s:0.093 data_s:0.014
873
  Training: 68%|█████████████████████████▊ | 16979/25000 [29:42<13:33, 9.86step/s]INFO 2026-05-20 18:08:39 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.06 loss:0.008 grdn:0.168 lr:1.2e-05 updt_s:0.093 data_s:0.007
874
- Training: 68%|█████████████████████████▊ | 16986/25000 [29:42<13:13, 10.09step/s]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
871
  Training: 68%|█████████████████████████▋ | 16940/25000 [29:37<13:13, 10.16step/s]INFO 2026-05-20 18:08:34 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.05 loss:0.008 grdn:0.197 lr:1.2e-05 updt_s:0.094 data_s:0.006
872
  Training: 68%|█████████████████████████▊ | 16960/25000 [29:40<14:37, 9.16step/s]INFO 2026-05-20 18:08:36 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.05 loss:0.008 grdn:0.186 lr:1.2e-05 updt_s:0.093 data_s:0.014
873
  Training: 68%|█████████████████████████▊ | 16979/25000 [29:42<13:33, 9.86step/s]INFO 2026-05-20 18:08:39 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.06 loss:0.008 grdn:0.168 lr:1.2e-05 updt_s:0.093 data_s:0.007
874
+ Training: 68%|█████████████████████████▊ | 16999/25000 [29:44<13:43, 9.72step/s]INFO 2026-05-20 18:08:41 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.06 loss:0.007 grdn:0.157 lr:1.2e-05 updt_s:0.090 data_s:0.008
875
+ Training: 68%|█████████████████████████▊ | 17019/25000 [29:46<12:57, 10.27step/s]INFO 2026-05-20 18:08:43 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.07 loss:0.008 grdn:0.164 lr:1.2e-05 updt_s:0.094 data_s:0.006
876
+ Training: 68%|█████████████████████████▉ | 17039/25000 [29:48<12:14, 10.84step/s]INFO 2026-05-20 18:08:45 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.07 loss:0.007 grdn:0.154 lr:1.2e-05 updt_s:0.090 data_s:0.008
877
+ Training: 68%|█████████████████████████▉ | 17059/25000 [29:50<13:47, 9.59step/s]INFO 2026-05-20 18:08:47 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.08 loss:0.007 grdn:0.153 lr:1.2e-05 updt_s:0.092 data_s:0.010
878
+ Training: 68%|█████████████████████████▉ | 17079/25000 [29:52<13:14, 9.97step/s]INFO 2026-05-20 18:08:49 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.08 loss:0.007 grdn:0.161 lr:1.2e-05 updt_s:0.088 data_s:0.008
879
+ Training: 68%|█████████████████████████▉ | 17100/25000 [29:54<12:39, 10.40step/s]INFO 2026-05-20 18:08:51 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.09 loss:0.008 grdn:0.176 lr:1.1e-05 updt_s:0.092 data_s:0.006
880
+ Training: 68%|██████████████████████████ | 17120/25000 [29:56<13:29, 9.74step/s]INFO 2026-05-20 18:08:53 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.09 loss:0.008 grdn:0.170 lr:1.1e-05 updt_s:0.092 data_s:0.009
881
+ Training: 69%|██████████████████████████ | 17140/25000 [29:58<13:13, 9.90step/s]INFO 2026-05-20 18:08:55 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.09 loss:0.008 grdn:0.180 lr:1.1e-05 updt_s:0.092 data_s:0.006
882
+ Training: 69%|██████████████████████████ | 17160/25000 [30:00<12:24, 10.54step/s]INFO 2026-05-20 18:08:56 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.10 loss:0.007 grdn:0.166 lr:1.1e-05 updt_s:0.088 data_s:0.006
883
+ Training: 69%|██████████████████████████ | 17179/25000 [30:01<12:31, 10.40step/s]INFO 2026-05-20 18:08:58 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.10 loss:0.007 grdn:0.163 lr:1.1e-05 updt_s:0.094 data_s:0.006
884
+ Training: 69%|██████████████████████████▏ | 17200/25000 [30:04<12:15, 10.60step/s]INFO 2026-05-20 18:09:00 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.11 loss:0.008 grdn:0.171 lr:1.1e-05 updt_s:0.088 data_s:0.009
885
+ Training: 69%|██████████████████████████▏ | 17219/25000 [30:06<12:51, 10.09step/s]INFO 2026-05-20 18:09:02 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.11 loss:0.008 grdn:0.179 lr:1.1e-05 updt_s:0.089 data_s:0.010
886
+ Training: 69%|██████████████████████████▏ | 17239/25000 [30:07<12:36, 10.25step/s]INFO 2026-05-20 18:09:04 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.12 loss:0.008 grdn:0.173 lr:1.1e-05 updt_s:0.088 data_s:0.011
887
+ Training: 69%|██████████████████████████▏ | 17259/25000 [30:09<12:43, 10.15step/s]INFO 2026-05-20 18:09:06 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.12 loss:0.009 grdn:0.187 lr:1.1e-05 updt_s:0.088 data_s:0.006
888
+ Training: 69%|██████████████████████████▎ | 17280/25000 [30:12<13:10, 9.76step/s]INFO 2026-05-20 18:09:08 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.13 loss:0.008 grdn:0.180 lr:1.1e-05 updt_s:0.089 data_s:0.006
889
+ Training: 69%|██████████████████████████▎ | 17300/25000 [30:14<12:31, 10.24step/s]INFO 2026-05-20 18:09:10 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.13 loss:0.007 grdn:0.163 lr:1.1e-05 updt_s:0.089 data_s:0.009
890
+ Training: 69%|██████████████████████████▎ | 17320/25000 [30:16<13:09, 9.73step/s]INFO 2026-05-20 18:09:12 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.14 loss:0.008 grdn:0.181 lr:1.1e-05 updt_s:0.089 data_s:0.009
891
+ Training: 69%|██████████████████████████▎ | 17339/25000 [30:17<12:33, 10.16step/s]INFO 2026-05-20 18:09:14 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.14 loss:0.008 grdn:0.173 lr:1.1e-05 updt_s:0.090 data_s:0.006
892
+ Training: 69%|██████████████████████████▍ | 17359/25000 [30:19<13:47, 9.23step/s]INFO 2026-05-20 18:09:16 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.15 loss:0.008 grdn:0.161 lr:1.1e-05 updt_s:0.092 data_s:0.007
893
+ Training: 70%|██████████████████████████▍ | 17379/25000 [30:21<12:16, 10.35step/s]INFO 2026-05-20 18:09:18 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.15 loss:0.007 grdn:0.159 lr:1.1e-05 updt_s:0.092 data_s:0.010
894
+ Training: 70%|██████████████████████████▍ | 17400/25000 [30:24<12:10, 10.41step/s]INFO 2026-05-20 18:09:20 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.16 loss:0.007 grdn:0.154 lr:1.1e-05 updt_s:0.087 data_s:0.011
895
+ Training: 70%|██████████████████████████▍ | 17419/25000 [30:26<12:47, 9.88step/s]INFO 2026-05-20 18:09:22 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.16 loss:0.008 grdn:0.157 lr:1.1e-05 updt_s:0.091 data_s:0.008
896
+ Training: 70%|██████████████████████████▌ | 17439/25000 [30:27<13:04, 9.64step/s]INFO 2026-05-20 18:09:24 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.17 loss:0.007 grdn:0.154 lr:1.1e-05 updt_s:0.092 data_s:0.008
897
+ Training: 70%|██████████████████████████▌ | 17460/25000 [30:30<11:57, 10.50step/s]INFO 2026-05-20 18:09:26 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.17 loss:0.008 grdn:0.180 lr:1.1e-05 updt_s:0.089 data_s:0.006
898
+ Training: 70%|██████████████████████████▌ | 17480/25000 [30:32<12:38, 9.92step/s]INFO 2026-05-20 18:09:28 ot_train.py:488 step:17K smpl:1M ep:3K epch:4.18 loss:0.008 grdn:0.172 lr:1.0e-05 updt_s:0.094 data_s:0.008
899
+ Training: 70%|██████████████████████████▌ | 17499/25000 [30:33<12:42, 9.84step/s]INFO 2026-05-20 18:09:30 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.18 loss:0.008 grdn:0.174 lr:1.0e-05 updt_s:0.090 data_s:0.006
900
+ Training: 70%|██████████████████████████▋ | 17519/25000 [30:36<13:57, 8.93step/s]INFO 2026-05-20 18:09:32 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.19 loss:0.008 grdn:0.173 lr:1.0e-05 updt_s:0.092 data_s:0.010
901
+ Training: 70%|██████████████████████████▋ | 17540/25000 [30:38<12:24, 10.02step/s]INFO 2026-05-20 18:09:34 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.19 loss:0.008 grdn:0.187 lr:1.0e-05 updt_s:0.089 data_s:0.007
902
+ Training: 70%|██████████████████████████▋ | 17559/25000 [30:39<11:49, 10.48step/s]INFO 2026-05-20 18:09:36 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.20 loss:0.008 grdn:0.173 lr:1.0e-05 updt_s:0.088 data_s:0.009
903
+ Training: 70%|██████████████████████████▋ | 17579/25000 [30:41<12:26, 9.95step/s]INFO 2026-05-20 18:09:38 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.20 loss:0.007 grdn:0.169 lr:1.0e-05 updt_s:0.087 data_s:0.011
904
+ Training: 70%|██████████████████████████▊ | 17599/25000 [30:43<12:11, 10.12step/s]INFO 2026-05-20 18:09:40 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.20 loss:0.008 grdn:0.183 lr:1.0e-05 updt_s:0.087 data_s:0.009
905
+ Training: 70%|██████████████████████████▊ | 17620/25000 [30:46<12:25, 9.90step/s]INFO 2026-05-20 18:09:42 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.21 loss:0.008 grdn:0.172 lr:1.0e-05 updt_s:0.088 data_s:0.012
906
+ Training: 71%|██████████████████████████▊ | 17639/25000 [30:47<12:34, 9.76step/s]INFO 2026-05-20 18:09:44 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.21 loss:0.006 grdn:0.147 lr:1.0e-05 updt_s:0.093 data_s:0.006
907
+ Training: 71%|██████████████████████████▊ | 17659/25000 [30:49<11:40, 10.47step/s]INFO 2026-05-20 18:09:46 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.22 loss:0.008 grdn:0.160 lr:1.0e-05 updt_s:0.086 data_s:0.009
908
+ Training: 71%|██████████████████████████▊ | 17680/25000 [30:51<11:02, 11.05step/s]INFO 2026-05-20 18:09:48 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.22 loss:0.008 grdn:0.155 lr:9.9e-06 updt_s:0.088 data_s:0.011
909
+ Training: 71%|██████████████████████████▉ | 17700/25000 [30:53<12:01, 10.11step/s]INFO 2026-05-20 18:09:50 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.23 loss:0.007 grdn:0.172 lr:9.9e-06 updt_s:0.094 data_s:0.006
910
+ Training: 71%|██████████████████████████▉ | 17720/25000 [30:55<11:32, 10.52step/s]INFO 2026-05-20 18:09:52 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.23 loss:0.008 grdn:0.165 lr:9.8e-06 updt_s:0.088 data_s:0.006
911
+ Training: 71%|██████████████████████████▉ | 17739/25000 [30:57<12:44, 9.49step/s]INFO 2026-05-20 18:09:54 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.24 loss:0.007 grdn:0.167 lr:9.8e-06 updt_s:0.091 data_s:0.009
912
+ Training: 71%|██████████████████████████▉ | 17760/25000 [30:59<12:29, 9.66step/s]INFO 2026-05-20 18:09:56 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.24 loss:0.008 grdn:0.190 lr:9.7e-06 updt_s:0.089 data_s:0.008
913
+ Training: 71%|███████████████████████████ | 17780/25000 [31:01<12:47, 9.41step/s]INFO 2026-05-20 18:09:58 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.25 loss:0.008 grdn:0.180 lr:9.7e-06 updt_s:0.088 data_s:0.008
914
+ Training: 71%|███████████████████████████ | 17800/25000 [31:03<11:56, 10.06step/s]INFO 2026-05-20 18:10:00 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.25 loss:0.008 grdn:0.177 lr:9.6e-06 updt_s:0.091 data_s:0.008
915
+ Training: 71%|███████████████████████████ | 17820/25000 [31:05<11:48, 10.13step/s]INFO 2026-05-20 18:10:02 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.26 loss:0.008 grdn:0.169 lr:9.6e-06 updt_s:0.091 data_s:0.006
916
+ Training: 71%|███████████████████████████ | 17840/25000 [31:07<11:49, 10.09step/s]INFO 2026-05-20 18:10:04 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.26 loss:0.008 grdn:0.180 lr:9.5e-06 updt_s:0.083 data_s:0.012
917
+ Training: 71%|███████████████████████████▏ | 17860/25000 [31:09<11:47, 10.10step/s]INFO 2026-05-20 18:10:06 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.27 loss:0.009 grdn:0.201 lr:9.5e-06 updt_s:0.090 data_s:0.009
918
+ Training: 72%|███████████████████████████▏ | 17880/25000 [31:11<11:16, 10.53step/s]INFO 2026-05-20 18:10:08 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.27 loss:0.009 grdn:0.198 lr:9.5e-06 updt_s:0.090 data_s:0.006
919
+ Training: 72%|███████████████████████████▏ | 17899/25000 [31:13<11:14, 10.52step/s]INFO 2026-05-20 18:10:10 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.28 loss:0.008 grdn:0.176 lr:9.4e-06 updt_s:0.091 data_s:0.006
920
+ Training: 72%|███████████████████████████▏ | 17919/25000 [31:15<12:32, 9.41step/s]INFO 2026-05-20 18:10:12 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.28 loss:0.008 grdn:0.176 lr:9.4e-06 updt_s:0.096 data_s:0.006
921
+ Training: 72%|███████████████████████████▎ | 17939/25000 [31:17<11:08, 10.56step/s]INFO 2026-05-20 18:10:14 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.29 loss:0.008 grdn:0.165 lr:9.3e-06 updt_s:0.086 data_s:0.006
922
+ Training: 72%|███████████████████████████▎ | 17959/25000 [31:19<11:25, 10.27step/s]INFO 2026-05-20 18:10:16 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.29 loss:0.008 grdn:0.166 lr:9.3e-06 updt_s:0.090 data_s:0.009
923
+ Training: 72%|███████████████████████████▎ | 17979/25000 [31:21<11:06, 10.53step/s]INFO 2026-05-20 18:10:18 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.30 loss:0.009 grdn:0.171 lr:9.2e-06 updt_s:0.091 data_s:0.008
924
+ Training: 72%|███████████████████████████▎ | 17999/25000 [31:23<11:08, 10.47step/s]INFO 2026-05-20 18:10:20 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.30 loss:0.008 grdn:0.163 lr:9.2e-06 updt_s:0.089 data_s:0.006
925
+ INFO 2026-05-20 18:10:20 ot_train.py:502 Checkpoint policy after step 18000
926
+ Training: 72%|███████████████████████████▍ | 18019/25000 [31:34<16:00, 7.27step/s]INFO 2026-05-20 18:10:31 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.30 loss:0.007 grdn:0.172 lr:9.1e-06 updt_s:0.070 data_s:0.005
927
+ Training: 72%|███████████████████████████▍ | 18040/25000 [31:36<11:44, 9.89step/s]INFO 2026-05-20 18:10:33 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.31 loss:0.008 grdn:0.166 lr:9.1e-06 updt_s:0.091 data_s:0.011
928
+ Training: 72%|███████████████████████████▍ | 18060/25000 [31:38<11:50, 9.77step/s]INFO 2026-05-20 18:10:35 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.31 loss:0.008 grdn:0.173 lr:9.0e-06 updt_s:0.089 data_s:0.009
929
+ Training: 72%|███████████████████████████▍ | 18079/25000 [31:40<11:35, 9.95step/s]INFO 2026-05-20 18:10:37 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.32 loss:0.007 grdn:0.170 lr:9.0e-06 updt_s:0.089 data_s:0.008
930
+ Training: 72%|███████████████████████████▌ | 18099/25000 [31:42<11:17, 10.18step/s]INFO 2026-05-20 18:10:39 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.32 loss:0.008 grdn:0.179 lr:8.9e-06 updt_s:0.088 data_s:0.010
931
+ Training: 72%|███████████████████████████▌ | 18119/25000 [31:44<11:25, 10.03step/s]INFO 2026-05-20 18:10:40 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.33 loss:0.008 grdn:0.180 lr:8.9e-06 updt_s:0.084 data_s:0.009
932
+ Training: 73%|███████████████████████████▌ | 18140/25000 [31:46<11:48, 9.68step/s]INFO 2026-05-20 18:10:42 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.33 loss:0.008 grdn:0.184 lr:8.8e-06 updt_s:0.090 data_s:0.009
933
+ Training: 73%|███████████████████████████▌ | 18160/25000 [31:48<10:31, 10.83step/s]INFO 2026-05-20 18:10:44 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.34 loss:0.007 grdn:0.173 lr:8.8e-06 updt_s:0.084 data_s:0.009
934
+ Training: 73%|███████████████████████████▋ | 18179/25000 [31:50<11:39, 9.76step/s]INFO 2026-05-20 18:10:46 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.34 loss:0.008 grdn:0.174 lr:8.7e-06 updt_s:0.089 data_s:0.010
935
+ Training: 73%|███████████████████████████▋ | 18199/25000 [31:51<11:11, 10.12step/s]INFO 2026-05-20 18:10:48 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.35 loss:0.007 grdn:0.177 lr:8.7e-06 updt_s:0.086 data_s:0.013
936
+ Training: 73%|███████████████████████████▋ | 18219/25000 [31:53<10:36, 10.65step/s]INFO 2026-05-20 18:10:50 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.35 loss:0.007 grdn:0.170 lr:8.6e-06 updt_s:0.090 data_s:0.009
937
+ Training: 73%|███████████████████████████▋ | 18239/25000 [31:55<11:33, 9.75step/s]INFO 2026-05-20 18:10:52 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.36 loss:0.008 grdn:0.182 lr:8.6e-06 updt_s:0.090 data_s:0.007
938
+ Training: 73%|███████████████████████████▊ | 18259/25000 [31:57<12:29, 8.99step/s]INFO 2026-05-20 18:10:54 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.36 loss:0.008 grdn:0.172 lr:8.5e-06 updt_s:0.093 data_s:0.008
939
+ Training: 73%|███████████████████████████▊ | 18279/25000 [31:59<10:17, 10.88step/s]INFO 2026-05-20 18:10:56 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.37 loss:0.009 grdn:0.186 lr:8.5e-06 updt_s:0.086 data_s:0.008
940
+ Training: 73%|███████████████████████████▊ | 18300/25000 [32:02<11:08, 10.02step/s]INFO 2026-05-20 18:10:58 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.37 loss:0.007 grdn:0.171 lr:8.4e-06 updt_s:0.093 data_s:0.008
941
+ Training: 73%|███████████████████████████▊ | 18320/25000 [32:03<11:52, 9.37step/s]INFO 2026-05-20 18:11:00 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.38 loss:0.007 grdn:0.167 lr:8.4e-06 updt_s:0.088 data_s:0.009
942
+ Training: 73%|███████████████████████████▉ | 18339/25000 [32:05<10:46, 10.30step/s]INFO 2026-05-20 18:11:02 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.38 loss:0.008 grdn:0.174 lr:8.3e-06 updt_s:0.091 data_s:0.008
943
+ Training: 73%|███████████████████████████▉ | 18359/25000 [32:07<11:20, 9.76step/s]INFO 2026-05-20 18:11:04 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.39 loss:0.008 grdn:0.161 lr:8.3e-06 updt_s:0.090 data_s:0.006
944
+ Training: 74%|███████████████████████████▉ | 18379/25000 [32:09<10:42, 10.31step/s]INFO 2026-05-20 18:11:06 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.39 loss:0.008 grdn:0.177 lr:8.2e-06 updt_s:0.088 data_s:0.009
945
+ Training: 74%|███████████████████████████▉ | 18399/25000 [32:11<10:55, 10.07step/s]INFO 2026-05-20 18:11:08 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.40 loss:0.008 grdn:0.166 lr:8.2e-06 updt_s:0.090 data_s:0.011
946
+ Training: 74%|███████████████████████████▉ | 18419/25000 [32:13<10:53, 10.07step/s]INFO 2026-05-20 18:11:10 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.40 loss:0.007 grdn:0.156 lr:8.2e-06 updt_s:0.089 data_s:0.006
947
+ Training: 74%|████████████████████████████ | 18439/25000 [32:15<10:18, 10.60step/s]INFO 2026-05-20 18:11:12 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.41 loss:0.008 grdn:0.183 lr:8.1e-06 updt_s:0.084 data_s:0.012
948
+ Training: 74%|████████████████████████████ | 18459/25000 [32:17<10:31, 10.36step/s]INFO 2026-05-20 18:11:14 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.41 loss:0.008 grdn:0.175 lr:8.1e-06 updt_s:0.087 data_s:0.012
949
+ Training: 74%|████████████████████████████ | 18479/25000 [32:19<10:55, 9.95step/s]INFO 2026-05-20 18:11:16 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.41 loss:0.008 grdn:0.188 lr:8.0e-06 updt_s:0.091 data_s:0.009
950
+ Training: 74%|████████████████████████████ | 18499/25000 [32:21<09:55, 10.91step/s]INFO 2026-05-20 18:11:18 ot_train.py:488 step:18K smpl:1M ep:3K epch:4.42 loss:0.008 grdn:0.196 lr:8.0e-06 updt_s:0.087 data_s:0.006
951
+ Training: 74%|████████████████████████████▏ | 18519/25000 [32:23<10:25, 10.36step/s]INFO 2026-05-20 18:11:20 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.42 loss:0.007 grdn:0.181 lr:7.9e-06 updt_s:0.091 data_s:0.010
952
+ Training: 74%|████████████████████████████▏ | 18539/25000 [32:25<11:33, 9.32step/s]INFO 2026-05-20 18:11:22 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.43 loss:0.008 grdn:0.176 lr:7.9e-06 updt_s:0.092 data_s:0.006
953
+ Training: 74%|████████████████████████████▏ | 18559/25000 [32:27<10:22, 10.34step/s]INFO 2026-05-20 18:11:24 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.43 loss:0.008 grdn:0.173 lr:7.8e-06 updt_s:0.088 data_s:0.007
954
+ Training: 74%|████████████████████████████▏ | 18580/25000 [32:29<11:09, 9.59step/s]INFO 2026-05-20 18:11:26 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.44 loss:0.008 grdn:0.182 lr:7.8e-06 updt_s:0.091 data_s:0.008
955
+ Training: 74%|████████████████████████████▎ | 18599/25000 [32:31<10:12, 10.44step/s]INFO 2026-05-20 18:11:28 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.44 loss:0.008 grdn:0.187 lr:7.7e-06 updt_s:0.088 data_s:0.008
956
+ Training: 74%|████████████████████████████▎ | 18619/25000 [32:33<11:38, 9.14step/s]INFO 2026-05-20 18:11:30 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.45 loss:0.008 grdn:0.180 lr:7.7e-06 updt_s:0.093 data_s:0.006
957
+ Training: 75%|████████████████████████████▎ | 18640/25000 [32:35<10:32, 10.06step/s]INFO 2026-05-20 18:11:32 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.45 loss:0.007 grdn:0.166 lr:7.6e-06 updt_s:0.089 data_s:0.011
958
+ Training: 75%|████████████████████████████▎ | 18660/25000 [32:37<10:39, 9.92step/s]INFO 2026-05-20 18:11:34 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.46 loss:0.008 grdn:0.167 lr:7.6e-06 updt_s:0.092 data_s:0.006
959
+ Training: 75%|████████████████████████████▍ | 18680/25000 [32:39<10:42, 9.84step/s]INFO 2026-05-20 18:11:36 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.46 loss:0.008 grdn:0.180 lr:7.6e-06 updt_s:0.089 data_s:0.011
960
+ Training: 75%|████████████████████████████▍ | 18699/25000 [32:41<10:15, 10.24step/s]INFO 2026-05-20 18:11:38 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.47 loss:0.008 grdn:0.175 lr:7.5e-06 updt_s:0.088 data_s:0.009
961
+ Training: 75%|████████████████████████████▍ | 18720/25000 [32:43<11:23, 9.19step/s]INFO 2026-05-20 18:11:40 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.47 loss:0.008 grdn:0.163 lr:7.5e-06 updt_s:0.092 data_s:0.008
962
+ Training: 75%|████████████████████████████▍ | 18739/25000 [32:45<10:40, 9.77step/s]INFO 2026-05-20 18:11:42 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.48 loss:0.007 grdn:0.188 lr:7.4e-06 updt_s:0.089 data_s:0.007
963
+ Training: 75%|████████████████████████████▌ | 18759/25000 [32:47<09:57, 10.44step/s]INFO 2026-05-20 18:11:44 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.48 loss:0.008 grdn:0.181 lr:7.4e-06 updt_s:0.088 data_s:0.009
964
+ Training: 75%|████████████████████████████▌ | 18779/25000 [32:49<10:12, 10.15step/s]INFO 2026-05-20 18:11:46 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.49 loss:0.007 grdn:0.160 lr:7.3e-06 updt_s:0.088 data_s:0.009
965
+ Training: 75%|████████████████████████████▌ | 18799/25000 [32:51<09:46, 10.58step/s]INFO 2026-05-20 18:11:48 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.49 loss:0.007 grdn:0.176 lr:7.3e-06 updt_s:0.094 data_s:0.006
966
+ Training: 75%|████████████████████████████▌ | 18819/25000 [32:53<10:59, 9.37step/s]INFO 2026-05-20 18:11:50 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.50 loss:0.008 grdn:0.180 lr:7.2e-06 updt_s:0.091 data_s:0.008
967
+ Training: 75%|████████████████████████████▋ | 18839/25000 [32:55<10:02, 10.23step/s]INFO 2026-05-20 18:11:52 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.50 loss:0.007 grdn:0.158 lr:7.2e-06 updt_s:0.085 data_s:0.011
968
+ Training: 75%|████████████████████████████▋ | 18859/25000 [32:57<11:20, 9.03step/s]INFO 2026-05-20 18:11:54 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.51 loss:0.008 grdn:0.181 lr:7.2e-06 updt_s:0.096 data_s:0.006
969
+ Training: 76%|████████████████████████████▋ | 18880/25000 [32:59<09:48, 10.40step/s]INFO 2026-05-20 18:11:56 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.51 loss:0.008 grdn:0.171 lr:7.1e-06 updt_s:0.085 data_s:0.008
970
+ Training: 76%|████████████████████████████▋ | 18899/25000 [33:01<09:52, 10.30step/s]INFO 2026-05-20 18:11:58 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.52 loss:0.008 grdn:0.169 lr:7.1e-06 updt_s:0.094 data_s:0.006
971
+ Training: 76%|████████████████████████████▊ | 18919/25000 [33:03<11:37, 8.72step/s]INFO 2026-05-20 18:12:00 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.52 loss:0.007 grdn:0.167 lr:7.0e-06 updt_s:0.094 data_s:0.007
972
+ Training: 76%|████████████████████████████▊ | 18939/25000 [33:05<09:43, 10.38step/s]INFO 2026-05-20 18:12:02 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.52 loss:0.008 grdn:0.173 lr:7.0e-06 updt_s:0.084 data_s:0.011
973
+ Training: 76%|████████████████████████████▊ | 18960/25000 [33:07<10:08, 9.92step/s]INFO 2026-05-20 18:12:04 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.53 loss:0.008 grdn:0.165 lr:6.9e-06 updt_s:0.091 data_s:0.011
974
+ Training: 76%|████████████████████████████▊ | 18979/25000 [33:09<09:52, 10.17step/s]INFO 2026-05-20 18:12:06 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.53 loss:0.007 grdn:0.166 lr:6.9e-06 updt_s:0.090 data_s:0.007
975
+ Training: 76%|████████████████████████████▉ | 18999/25000 [33:11<09:40, 10.34step/s]INFO 2026-05-20 18:12:08 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.54 loss:0.008 grdn:0.175 lr:6.8e-06 updt_s:0.090 data_s:0.008
976
+ Training: 76%|████████████████████████████▉ | 19019/25000 [33:13<09:58, 9.99step/s]INFO 2026-05-20 18:12:10 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.54 loss:0.008 grdn:0.171 lr:6.8e-06 updt_s:0.094 data_s:0.006
977
+ Training: 76%|████████████████████████████▉ | 19039/25000 [33:15<10:18, 9.64step/s]INFO 2026-05-20 18:12:12 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.55 loss:0.008 grdn:0.181 lr:6.8e-06 updt_s:0.087 data_s:0.009
978
+ Training: 76%|████████████████████████████▉ | 19059/25000 [33:17<09:48, 10.09step/s]INFO 2026-05-20 18:12:14 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.55 loss:0.008 grdn:0.171 lr:6.7e-06 updt_s:0.088 data_s:0.009
979
+ Training: 76%|█████████████████████████████ | 19079/25000 [33:19<10:09, 9.72step/s]INFO 2026-05-20 18:12:16 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.56 loss:0.008 grdn:0.178 lr:6.7e-06 updt_s:0.091 data_s:0.008
980
+ Training: 76%|█████████████████████████████ | 19099/25000 [33:21<10:26, 9.41step/s]INFO 2026-05-20 18:12:18 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.56 loss:0.008 grdn:0.178 lr:6.6e-06 updt_s:0.088 data_s:0.009
981
+ Training: 76%|█████████████████████████████ | 19119/25000 [33:23<08:56, 10.97step/s]INFO 2026-05-20 18:12:20 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.57 loss:0.008 grdn:0.172 lr:6.6e-06 updt_s:0.086 data_s:0.009
982
+ Training: 77%|█████████████████████████████ | 19139/25000 [33:25<09:47, 9.97step/s]INFO 2026-05-20 18:12:22 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.57 loss:0.007 grdn:0.162 lr:6.5e-06 updt_s:0.090 data_s:0.009
983
+ Training: 77%|█████████████████████████████ | 19159/25000 [33:27<09:36, 10.13step/s]INFO 2026-05-20 18:12:24 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.58 loss:0.007 grdn:0.169 lr:6.5e-06 updt_s:0.089 data_s:0.007
984
+ Training: 77%|█████████████████████████████▏ | 19180/25000 [33:29<09:57, 9.75step/s]INFO 2026-05-20 18:12:26 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.58 loss:0.007 grdn:0.170 lr:6.5e-06 updt_s:0.089 data_s:0.009
985
+ Training: 77%|█████████████████████████████▏ | 19199/25000 [33:31<09:13, 10.49step/s]INFO 2026-05-20 18:12:28 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.59 loss:0.007 grdn:0.166 lr:6.4e-06 updt_s:0.088 data_s:0.006
986
+ Training: 77%|█████████████████████████████▏ | 19220/25000 [33:33<09:23, 10.26step/s]INFO 2026-05-20 18:12:30 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.59 loss:0.008 grdn:0.179 lr:6.4e-06 updt_s:0.091 data_s:0.009
987
+ Training: 77%|█████████████████████████████▏ | 19240/25000 [33:35<08:54, 10.78step/s]INFO 2026-05-20 18:12:32 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.60 loss:0.007 grdn:0.176 lr:6.3e-06 updt_s:0.085 data_s:0.009
988
+ Training: 77%|█████████████████████████████▎ | 19260/25000 [33:37<09:30, 10.05step/s]INFO 2026-05-20 18:12:34 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.60 loss:0.007 grdn:0.166 lr:6.3e-06 updt_s:0.094 data_s:0.006
989
+ Training: 77%|█████████████████████████████▎ | 19280/25000 [33:39<09:51, 9.67step/s]INFO 2026-05-20 18:12:36 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.61 loss:0.008 grdn:0.187 lr:6.3e-06 updt_s:0.091 data_s:0.008
990
+ Training: 77%|█████████████████████████████▎ | 19299/25000 [33:41<08:47, 10.81step/s]INFO 2026-05-20 18:12:38 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.61 loss:0.008 grdn:0.174 lr:6.2e-06 updt_s:0.084 data_s:0.010
991
+ Training: 77%|█████████████████████████████▎ | 19319/25000 [33:43<09:17, 10.20step/s]INFO 2026-05-20 18:12:40 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.62 loss:0.008 grdn:0.171 lr:6.2e-06 updt_s:0.089 data_s:0.010
992
+ Training: 77%|█████████████████████████████▍ | 19339/25000 [33:45<09:08, 10.32step/s]INFO 2026-05-20 18:12:42 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.62 loss:0.008 grdn:0.166 lr:6.1e-06 updt_s:0.085 data_s:0.013
993
+ Training: 77%|█████████████████████████████▍ | 19359/25000 [33:47<09:00, 10.43step/s]INFO 2026-05-20 18:12:44 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.63 loss:0.008 grdn:0.183 lr:6.1e-06 updt_s:0.090 data_s:0.006
994
+ Training: 78%|█████████████████████████████▍ | 19380/25000 [33:49<09:01, 10.38step/s]INFO 2026-05-20 18:12:46 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.63 loss:0.008 grdn:0.174 lr:6.0e-06 updt_s:0.092 data_s:0.008
995
+ Training: 78%|█████████████████████████████▍ | 19400/25000 [33:51<09:45, 9.56step/s]INFO 2026-05-20 18:12:48 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.63 loss:0.007 grdn:0.156 lr:6.0e-06 updt_s:0.090 data_s:0.010
996
+ Training: 78%|█████████████████████████████▌ | 19420/25000 [33:53<08:59, 10.35step/s]INFO 2026-05-20 18:12:50 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.64 loss:0.008 grdn:0.178 lr:6.0e-06 updt_s:0.089 data_s:0.006
997
+ Training: 78%|█████████████████████████████▌ | 19440/25000 [33:55<09:28, 9.78step/s]INFO 2026-05-20 18:12:52 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.64 loss:0.007 grdn:0.162 lr:5.9e-06 updt_s:0.089 data_s:0.011
998
+ Training: 78%|█████████████████████████████▌ | 19460/25000 [33:57<09:16, 9.96step/s]INFO 2026-05-20 18:12:53 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.65 loss:0.008 grdn:0.188 lr:5.9e-06 updt_s:0.090 data_s:0.006
999
+ Training: 78%|█████████████████████████████▌ | 19480/25000 [33:59<08:45, 10.50step/s]INFO 2026-05-20 18:12:55 ot_train.py:488 step:19K smpl:1M ep:3K epch:4.65 loss:0.007 grdn:0.162 lr:5.8e-06 updt_s:0.089 data_s:0.006
1000
+ Training: 78%|█████████████████████████████▋ | 19499/25000 [34:00<09:09, 10.01step/s]INFO 2026-05-20 18:12:57 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.66 loss:0.007 grdn:0.160 lr:5.8e-06 updt_s:0.092 data_s:0.006
1001
+ Training: 78%|█████████████████████████████▋ | 19519/25000 [34:02<09:00, 10.14step/s]INFO 2026-05-20 18:12:59 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.66 loss:0.008 grdn:0.168 lr:5.8e-06 updt_s:0.085 data_s:0.012
1002
+ Training: 78%|█████████████████████████████▋ | 19540/25000 [34:04<08:38, 10.53step/s]INFO 2026-05-20 18:13:01 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.67 loss:0.007 grdn:0.173 lr:5.7e-06 updt_s:0.090 data_s:0.006
1003
+ Training: 78%|█████████████████████████████▋ | 19560/25000 [34:06<09:10, 9.89step/s]INFO 2026-05-20 18:13:03 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.67 loss:0.008 grdn:0.186 lr:5.7e-06 updt_s:0.088 data_s:0.009
1004
+ Training: 78%|█████████████████████████████▊ | 19580/25000 [34:08<09:07, 9.90step/s]INFO 2026-05-20 18:13:05 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.68 loss:0.008 grdn:0.180 lr:5.6e-06 updt_s:0.086 data_s:0.009
1005
+ Training: 78%|█████████████████████████████▊ | 19600/25000 [34:10<08:39, 10.40step/s]INFO 2026-05-20 18:13:07 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.68 loss:0.007 grdn:0.163 lr:5.6e-06 updt_s:0.090 data_s:0.008
1006
+ Training: 78%|█████████████████████████████▊ | 19620/25000 [34:12<08:59, 9.97step/s]INFO 2026-05-20 18:13:09 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.69 loss:0.008 grdn:0.173 lr:5.6e-06 updt_s:0.087 data_s:0.009
1007
+ Training: 79%|█████████████████████████████▊ | 19640/25000 [34:14<08:21, 10.70step/s]INFO 2026-05-20 18:13:11 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.69 loss:0.008 grdn:0.179 lr:5.5e-06 updt_s:0.088 data_s:0.008
1008
+ Training: 79%|█████████████████████████████▉ | 19659/25000 [34:16<09:52, 9.02step/s]INFO 2026-05-20 18:13:13 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.70 loss:0.007 grdn:0.162 lr:5.5e-06 updt_s:0.091 data_s:0.011
1009
+ Training: 79%|█████████████████████████████▉ | 19680/25000 [34:18<08:29, 10.43step/s]INFO 2026-05-20 18:13:15 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.70 loss:0.007 grdn:0.155 lr:5.4e-06 updt_s:0.087 data_s:0.011
1010
+ Training: 79%|█████████████████████████████▉ | 19700/25000 [34:20<08:43, 10.13step/s]INFO 2026-05-20 18:13:17 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.71 loss:0.007 grdn:0.170 lr:5.4e-06 updt_s:0.090 data_s:0.006
1011
+ Training: 79%|█████████████████████████████▉ | 19720/25000 [34:22<08:56, 9.85step/s]INFO 2026-05-20 18:13:19 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.71 loss:0.008 grdn:0.170 lr:5.4e-06 updt_s:0.092 data_s:0.006
1012
+ Training: 79%|██████████████████████████████ | 19740/25000 [34:24<08:46, 9.99step/s]INFO 2026-05-20 18:13:21 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.72 loss:0.009 grdn:0.182 lr:5.3e-06 updt_s:0.087 data_s:0.011
1013
+ Training: 79%|██████████████████████████████ | 19759/25000 [34:26<08:38, 10.11step/s]INFO 2026-05-20 18:13:23 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.72 loss:0.008 grdn:0.181 lr:5.3e-06 updt_s:0.089 data_s:0.012
1014
+ Training: 79%|██████████████████████████████ | 19779/25000 [34:28<07:58, 10.91step/s]INFO 2026-05-20 18:13:25 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.73 loss:0.009 grdn:0.200 lr:5.2e-06 updt_s:0.089 data_s:0.007
1015
+ Training: 79%|██████████████████████████████ | 19799/25000 [34:30<08:01, 10.79step/s]INFO 2026-05-20 18:13:27 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.73 loss:0.008 grdn:0.171 lr:5.2e-06 updt_s:0.092 data_s:0.008
1016
+ Training: 79%|██████████████████████████████ | 19819/25000 [34:32<08:40, 9.95step/s]INFO 2026-05-20 18:13:29 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.73 loss:0.008 grdn:0.174 lr:5.2e-06 updt_s:0.091 data_s:0.008
1017
+ Training: 79%|██████████████████████████████▏ | 19839/25000 [34:34<08:23, 10.25step/s]INFO 2026-05-20 18:13:31 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.74 loss:0.008 grdn:0.186 lr:5.1e-06 updt_s:0.086 data_s:0.011
1018
+ Training: 79%|██████████████████████████████▏ | 19860/25000 [34:36<08:46, 9.77step/s]INFO 2026-05-20 18:13:33 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.74 loss:0.008 grdn:0.174 lr:5.1e-06 updt_s:0.087 data_s:0.008
1019
+ Training: 80%|██████████████████████████████▏ | 19879/25000 [34:38<08:25, 10.12step/s]INFO 2026-05-20 18:13:35 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.75 loss:0.006 grdn:0.158 lr:5.1e-06 updt_s:0.092 data_s:0.008
1020
+ Training: 80%|██████████████████████████████▏ | 19899/25000 [34:40<08:18, 10.23step/s]INFO 2026-05-20 18:13:37 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.75 loss:0.008 grdn:0.180 lr:5.0e-06 updt_s:0.087 data_s:0.009
1021
+ Training: 80%|██████████████████████████████▎ | 19920/25000 [34:42<08:55, 9.49step/s]INFO 2026-05-20 18:13:39 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.76 loss:0.006 grdn:0.156 lr:5.0e-06 updt_s:0.088 data_s:0.008
1022
+ Training: 80%|██████████████████████████████▎ | 19939/25000 [34:44<08:25, 10.01step/s]INFO 2026-05-20 18:13:41 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.76 loss:0.008 grdn:0.187 lr:4.9e-06 updt_s:0.089 data_s:0.011
1023
+ Training: 80%|██████████████████████████████▎ | 19959/25000 [34:46<08:16, 10.16step/s]INFO 2026-05-20 18:13:43 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.77 loss:0.007 grdn:0.186 lr:4.9e-06 updt_s:0.089 data_s:0.009
1024
+ Training: 80%|██████████████████████████████▎ | 19980/25000 [34:48<08:38, 9.68step/s]INFO 2026-05-20 18:13:45 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.77 loss:0.007 grdn:0.172 lr:4.9e-06 updt_s:0.089 data_s:0.010
1025
+ Training: 80%|██████████████████████████████▍ | 19999/25000 [34:50<08:47, 9.48step/s]INFO 2026-05-20 18:13:47 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.78 loss:0.007 grdn:0.177 lr:4.8e-06 updt_s:0.091 data_s:0.008
1026
+ INFO 2026-05-20 18:13:47 ot_train.py:502 Checkpoint policy after step 20000
1027
+ Training: 80%|██████████████████████████████▍ | 20019/25000 [35:01<11:36, 7.16step/s]INFO 2026-05-20 18:13:58 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.78 loss:0.007 grdn:0.158 lr:4.8e-06 updt_s:0.071 data_s:0.005
1028
+ Training: 80%|██████████████████████████████▍ | 20040/25000 [35:03<07:36, 10.86step/s]INFO 2026-05-20 18:14:00 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.79 loss:0.008 grdn:0.183 lr:4.8e-06 updt_s:0.088 data_s:0.009
1029
+ Training: 80%|██████████████████████████████▍ | 20059/25000 [35:05<08:20, 9.87step/s]INFO 2026-05-20 18:14:02 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.79 loss:0.008 grdn:0.183 lr:4.7e-06 updt_s:0.092 data_s:0.009
1030
+ Training: 80%|██████████████████████████████▌ | 20079/25000 [35:07<08:29, 9.65step/s]INFO 2026-05-20 18:14:04 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.80 loss:0.007 grdn:0.172 lr:4.7e-06 updt_s:0.087 data_s:0.010
1031
+ Training: 80%|██████████████████████████████▌ | 20099/25000 [35:09<07:59, 10.23step/s]INFO 2026-05-20 18:14:06 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.80 loss:0.008 grdn:0.167 lr:4.6e-06 updt_s:0.084 data_s:0.009
1032
+ Training: 80%|██████████████████████████████▌ | 20120/25000 [35:11<08:11, 9.94step/s]INFO 2026-05-20 18:14:08 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.81 loss:0.009 grdn:0.192 lr:4.6e-06 updt_s:0.088 data_s:0.013
1033
+ Training: 81%|██████████████████████████████▌ | 20140/25000 [35:13<08:00, 10.12step/s]INFO 2026-05-20 18:14:10 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.81 loss:0.008 grdn:0.196 lr:4.6e-06 updt_s:0.087 data_s:0.008
1034
+ Training: 81%|██████████████████████████████▋ | 20160/25000 [35:15<07:58, 10.12step/s]INFO 2026-05-20 18:14:12 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.82 loss:0.007 grdn:0.165 lr:4.5e-06 updt_s:0.086 data_s:0.010
1035
+ Training: 81%|██████████████████████████████▋ | 20179/25000 [35:17<08:20, 9.63step/s]INFO 2026-05-20 18:14:14 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.82 loss:0.008 grdn:0.182 lr:4.5e-06 updt_s:0.093 data_s:0.006
1036
+ Training: 81%|██████████████████████████████▋ | 20200/25000 [35:19<07:41, 10.39step/s]INFO 2026-05-20 18:14:16 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.83 loss:0.008 grdn:0.171 lr:4.5e-06 updt_s:0.089 data_s:0.008
1037
+ Training: 81%|██████████████████████████████▋ | 20220/25000 [35:21<07:21, 10.83step/s]INFO 2026-05-20 18:14:18 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.83 loss:0.008 grdn:0.185 lr:4.4e-06 updt_s:0.085 data_s:0.009
1038
+ Training: 81%|██████████████████████████████▊ | 20240/25000 [35:23<08:02, 9.87step/s]INFO 2026-05-20 18:14:20 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.84 loss:0.008 grdn:0.180 lr:4.4e-06 updt_s:0.096 data_s:0.006
1039
+ Training: 81%|██████████████████████████████▊ | 20260/25000 [35:25<07:26, 10.61step/s]INFO 2026-05-20 18:14:22 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.84 loss:0.007 grdn:0.166 lr:4.4e-06 updt_s:0.089 data_s:0.007
1040
+ Training: 81%|██████████████████████████████▊ | 20280/25000 [35:27<07:39, 10.28step/s]INFO 2026-05-20 18:14:24 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.84 loss:0.007 grdn:0.165 lr:4.3e-06 updt_s:0.087 data_s:0.009
1041
+ Training: 81%|██████████████████████████████▊ | 20300/25000 [35:29<07:34, 10.33step/s]INFO 2026-05-20 18:14:26 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.85 loss:0.007 grdn:0.179 lr:4.3e-06 updt_s:0.090 data_s:0.010
1042
+ Training: 81%|██████████████████████████████▉ | 20319/25000 [35:31<08:03, 9.68step/s]INFO 2026-05-20 18:14:28 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.85 loss:0.007 grdn:0.170 lr:4.2e-06 updt_s:0.093 data_s:0.006
1043
+ Training: 81%|██████████████████████████████▉ | 20340/25000 [35:33<08:06, 9.57step/s]INFO 2026-05-20 18:14:30 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.86 loss:0.008 grdn:0.178 lr:4.2e-06 updt_s:0.089 data_s:0.009
1044
+ Training: 81%|██████████████████████████████▉ | 20359/25000 [35:35<07:19, 10.56step/s]INFO 2026-05-20 18:14:32 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.86 loss:0.007 grdn:0.167 lr:4.2e-06 updt_s:0.091 data_s:0.006
1045
+ Training: 82%|██████████████████████████████▉ | 20380/25000 [35:37<08:16, 9.30step/s]INFO 2026-05-20 18:14:34 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.87 loss:0.009 grdn:0.185 lr:4.1e-06 updt_s:0.095 data_s:0.007
1046
+ Training: 82%|███████████████████████████████ | 20399/25000 [35:39<07:53, 9.72step/s]INFO 2026-05-20 18:14:36 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.87 loss:0.008 grdn:0.183 lr:4.1e-06 updt_s:0.089 data_s:0.009
1047
+ Training: 82%|███████████████████████████████ | 20420/25000 [35:41<07:30, 10.16step/s]INFO 2026-05-20 18:14:38 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.88 loss:0.007 grdn:0.161 lr:4.1e-06 updt_s:0.092 data_s:0.006
1048
+ Training: 82%|███████████████████████████████ | 20440/25000 [35:43<07:35, 10.01step/s]INFO 2026-05-20 18:14:40 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.88 loss:0.009 grdn:0.190 lr:4.0e-06 updt_s:0.091 data_s:0.006
1049
+ Training: 82%|███████████████████████████████ | 20460/25000 [35:45<07:52, 9.60step/s]INFO 2026-05-20 18:14:42 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.89 loss:0.008 grdn:0.179 lr:4.0e-06 updt_s:0.094 data_s:0.006
1050
+ Training: 82%|███████████████████████████████▏ | 20480/25000 [35:47<07:16, 10.34step/s]INFO 2026-05-20 18:14:44 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.89 loss:0.009 grdn:0.186 lr:4.0e-06 updt_s:0.089 data_s:0.006
1051
+ Training: 82%|███████████████████████████████▏ | 20499/25000 [35:49<06:42, 11.20step/s]INFO 2026-05-20 18:14:46 ot_train.py:488 step:20K smpl:1M ep:3K epch:4.90 loss:0.008 grdn:0.172 lr:3.9e-06 updt_s:0.089 data_s:0.006
1052
+ Training: 82%|███████████████████████████████▏ | 20519/25000 [35:51<07:18, 10.22step/s]INFO 2026-05-20 18:14:48 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.90 loss:0.007 grdn:0.170 lr:3.9e-06 updt_s:0.090 data_s:0.009
1053
+ Training: 82%|███████████████████████████████▏ | 20539/25000 [35:53<06:03, 12.26step/s]INFO 2026-05-20 18:14:50 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.91 loss:0.007 grdn:0.159 lr:3.9e-06 updt_s:0.080 data_s:0.021
1054
+ Training: 82%|███████████████████████████████▏ | 20559/25000 [35:54<05:19, 13.91step/s]INFO 2026-05-20 18:14:51 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.91 loss:0.009 grdn:0.184 lr:3.8e-06 updt_s:0.066 data_s:0.005
1055
+ Training: 82%|███████████████████████████████▎ | 20579/25000 [35:56<05:17, 13.93step/s]INFO 2026-05-20 18:14:53 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.92 loss:0.008 grdn:0.180 lr:3.8e-06 updt_s:0.067 data_s:0.005
1056
+ Training: 82%|███████████████████████████████▎ | 20599/25000 [36:00<13:33, 5.41step/s]INFO 2026-05-20 18:14:57 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.92 loss:0.008 grdn:0.185 lr:3.8e-06 updt_s:0.075 data_s:0.132
1057
+ Training: 82%|███████████████████████████████▎ | 20620/25000 [36:02<08:32, 8.55step/s]INFO 2026-05-20 18:14:59 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.93 loss:0.009 grdn:0.175 lr:3.7e-06 updt_s:0.097 data_s:0.010
1058
+ Training: 83%|███████████████████████████████▎ | 20639/25000 [36:04<06:52, 10.57step/s]INFO 2026-05-20 18:15:01 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.93 loss:0.007 grdn:0.166 lr:3.7e-06 updt_s:0.090 data_s:0.008
1059
+ Training: 83%|███████████████████████████████▍ | 20659/25000 [36:06<08:08, 8.89step/s]INFO 2026-05-20 18:15:03 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.94 loss:0.008 grdn:0.182 lr:3.7e-06 updt_s:0.096 data_s:0.010
1060
+ Training: 83%|███████████████████████████████▍ | 20680/25000 [36:08<08:08, 8.84step/s]INFO 2026-05-20 18:15:05 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.94 loss:0.007 grdn:0.172 lr:3.6e-06 updt_s:0.098 data_s:0.008
1061
+ Training: 83%|███████████████████████████████▍ | 20700/25000 [36:11<07:05, 10.10step/s]INFO 2026-05-20 18:15:07 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.95 loss:0.008 grdn:0.173 lr:3.6e-06 updt_s:0.096 data_s:0.011
1062
+ Training: 83%|███████████████████████████████▍ | 20719/25000 [36:13<08:12, 8.70step/s]INFO 2026-05-20 18:15:10 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.95 loss:0.008 grdn:0.170 lr:3.6e-06 updt_s:0.099 data_s:0.011
1063
+ Training: 83%|███████████████████████████████▌ | 20740/25000 [36:15<07:53, 9.00step/s]INFO 2026-05-20 18:15:12 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.95 loss:0.007 grdn:0.165 lr:3.5e-06 updt_s:0.097 data_s:0.009
1064
+ Training: 83%|███████████████████████████████▌ | 20759/25000 [36:17<06:57, 10.16step/s]INFO 2026-05-20 18:15:14 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.96 loss:0.008 grdn:0.167 lr:3.5e-06 updt_s:0.087 data_s:0.012
1065
+ Training: 83%|███████████████████████████████▌ | 20780/25000 [36:19<06:53, 10.20step/s]INFO 2026-05-20 18:15:16 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.96 loss:0.007 grdn:0.162 lr:3.5e-06 updt_s:0.093 data_s:0.006
1066
+ Training: 83%|███████████████████████████████▌ | 20799/25000 [36:21<07:23, 9.47step/s]INFO 2026-05-20 18:15:18 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.97 loss:0.007 grdn:0.171 lr:3.4e-06 updt_s:0.094 data_s:0.010
1067
+ Training: 83%|███████████████████████████████▋ | 20819/25000 [36:23<06:31, 10.68step/s]INFO 2026-05-20 18:15:20 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.97 loss:0.008 grdn:0.174 lr:3.4e-06 updt_s:0.090 data_s:0.008
1068
+ Training: 83%|███████████████████████████████▋ | 20840/25000 [36:25<07:18, 9.48step/s]INFO 2026-05-20 18:15:22 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.98 loss:0.009 grdn:0.192 lr:3.4e-06 updt_s:0.090 data_s:0.009
1069
+ Training: 83%|███���███████████████████████████▋ | 20860/25000 [36:27<06:45, 10.22step/s]INFO 2026-05-20 18:15:24 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.98 loss:0.008 grdn:0.184 lr:3.3e-06 updt_s:0.091 data_s:0.011
1070
+ Training: 84%|███████████████████████████████▋ | 20880/25000 [36:29<06:24, 10.71step/s]INFO 2026-05-20 18:15:26 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.99 loss:0.008 grdn:0.185 lr:3.3e-06 updt_s:0.090 data_s:0.009
1071
+ Training: 84%|███████████████████████████████▊ | 20900/25000 [36:31<06:45, 10.11step/s]INFO 2026-05-20 18:15:28 ot_train.py:488 step:21K smpl:1M ep:3K epch:4.99 loss:0.007 grdn:0.163 lr:3.3e-06 updt_s:0.089 data_s:0.008
1072
+ Training: 84%|███████████████████████████████▊ | 20920/25000 [36:33<06:47, 10.01step/s]INFO 2026-05-20 18:15:30 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.00 loss:0.008 grdn:0.166 lr:3.3e-06 updt_s:0.095 data_s:0.006
1073
+ Training: 84%|███████████████████████████████▊ | 20939/25000 [36:35<06:42, 10.08step/s]INFO 2026-05-20 18:15:32 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.00 loss:0.008 grdn:0.182 lr:3.2e-06 updt_s:0.091 data_s:0.008
1074
+ Training: 84%|███████████████████████████████▊ | 20960/25000 [36:37<07:03, 9.55step/s]INFO 2026-05-20 18:15:34 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.01 loss:0.008 grdn:0.165 lr:3.2e-06 updt_s:0.088 data_s:0.014
1075
+ Training: 84%|███████████████████████████████▉ | 20980/25000 [36:39<06:18, 10.61step/s]INFO 2026-05-20 18:15:36 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.01 loss:0.007 grdn:0.170 lr:3.2e-06 updt_s:0.091 data_s:0.006
1076
+ Training: 84%|███████████████████████████████▉ | 20999/25000 [36:41<06:45, 9.87step/s]INFO 2026-05-20 18:15:38 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.02 loss:0.007 grdn:0.176 lr:3.1e-06 updt_s:0.087 data_s:0.011
1077
+ Training: 84%|███████████████████████████████▉ | 21020/25000 [36:43<07:02, 9.43step/s]INFO 2026-05-20 18:15:40 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.02 loss:0.007 grdn:0.176 lr:3.1e-06 updt_s:0.095 data_s:0.008
1078
+ Training: 84%|███████████████████████████████▉ | 21040/25000 [36:45<06:11, 10.65step/s]INFO 2026-05-20 18:15:42 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.03 loss:0.009 grdn:0.190 lr:3.1e-06 updt_s:0.085 data_s:0.011
1079
+ Training: 84%|████████████████████████████████ | 21059/25000 [36:47<06:10, 10.65step/s]INFO 2026-05-20 18:15:44 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.03 loss:0.007 grdn:0.173 lr:3.0e-06 updt_s:0.087 data_s:0.008
1080
+ Training: 84%|████████████████████████████████ | 21079/25000 [36:49<06:26, 10.14step/s]INFO 2026-05-20 18:15:46 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.04 loss:0.008 grdn:0.170 lr:3.0e-06 updt_s:0.087 data_s:0.010
1081
+ Training: 84%|████████████████████████████████ | 21099/25000 [36:51<06:45, 9.62step/s]INFO 2026-05-20 18:15:48 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.04 loss:0.008 grdn:0.181 lr:3.0e-06 updt_s:0.092 data_s:0.007
1082
+ Training: 84%|████████████████████████████████ | 21120/25000 [36:53<06:04, 10.66step/s]INFO 2026-05-20 18:15:50 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.05 loss:0.008 grdn:0.198 lr:3.0e-06 updt_s:0.092 data_s:0.006
1083
+ Training: 85%|████████████████████████████████▏ | 21140/25000 [36:55<06:37, 9.70step/s]INFO 2026-05-20 18:15:52 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.05 loss:0.008 grdn:0.179 lr:2.9e-06 updt_s:0.090 data_s:0.013
1084
+ Training: 85%|████████████████████████████████▏ | 21160/25000 [36:57<06:03, 10.57step/s]INFO 2026-05-20 18:15:54 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.06 loss:0.009 grdn:0.201 lr:2.9e-06 updt_s:0.088 data_s:0.009
1085
+ Training: 85%|████████████████████████████████▏ | 21180/25000 [36:59<06:25, 9.90step/s]INFO 2026-05-20 18:15:56 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.06 loss:0.008 grdn:0.176 lr:2.9e-06 updt_s:0.091 data_s:0.009
1086
+ Training: 85%|████████████████████████████████▏ | 21200/25000 [37:01<06:19, 10.03step/s]INFO 2026-05-20 18:15:58 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.06 loss:0.007 grdn:0.176 lr:2.8e-06 updt_s:0.088 data_s:0.009
1087
+ Training: 85%|████████████████████████████████▎ | 21219/25000 [37:03<06:41, 9.41step/s]INFO 2026-05-20 18:16:00 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.07 loss:0.008 grdn:0.188 lr:2.8e-06 updt_s:0.091 data_s:0.011
1088
+ Training: 85%|████████████████████████████████▎ | 21239/25000 [37:05<06:14, 10.04step/s]INFO 2026-05-20 18:16:02 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.07 loss:0.007 grdn:0.170 lr:2.8e-06 updt_s:0.088 data_s:0.008
1089
+ Training: 85%|████████████████████████████████▎ | 21259/25000 [37:07<06:10, 10.11step/s]INFO 2026-05-20 18:16:04 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.08 loss:0.007 grdn:0.158 lr:2.7e-06 updt_s:0.090 data_s:0.009
1090
+ Training: 85%|████████████████████████████████▎ | 21280/25000 [37:09<06:21, 9.74step/s]INFO 2026-05-20 18:16:06 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.08 loss:0.008 grdn:0.175 lr:2.7e-06 updt_s:0.094 data_s:0.009
1091
+ Training: 85%|████████████████████████████████▍ | 21300/25000 [37:11<06:43, 9.17step/s]INFO 2026-05-20 18:16:08 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.09 loss:0.008 grdn:0.178 lr:2.7e-06 updt_s:0.086 data_s:0.013
1092
+ Training: 85%|████████████████████████████████▍ | 21320/25000 [37:13<06:03, 10.14step/s]INFO 2026-05-20 18:16:10 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.09 loss:0.007 grdn:0.164 lr:2.7e-06 updt_s:0.091 data_s:0.006
1093
+ Training: 85%|████████████████████████████████▍ | 21339/25000 [37:15<06:21, 9.60step/s]INFO 2026-05-20 18:16:12 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.10 loss:0.007 grdn:0.166 lr:2.6e-06 updt_s:0.092 data_s:0.008
1094
+ Training: 85%|████████████████████████████████▍ | 21359/25000 [37:17<05:43, 10.60step/s]INFO 2026-05-20 18:16:14 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.10 loss:0.007 grdn:0.162 lr:2.6e-06 updt_s:0.087 data_s:0.007
1095
+ Training: 86%|████████████████████████████████▍ | 21379/25000 [37:19<06:29, 9.29step/s]INFO 2026-05-20 18:16:16 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.11 loss:0.008 grdn:0.180 lr:2.6e-06 updt_s:0.092 data_s:0.008
1096
+ Training: 86%|████████████████████████████████▌ | 21400/25000 [37:21<05:53, 10.19step/s]INFO 2026-05-20 18:16:18 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.11 loss:0.008 grdn:0.176 lr:2.5e-06 updt_s:0.091 data_s:0.008
1097
+ Training: 86%|████████████████████████████████▌ | 21420/25000 [37:23<05:44, 10.39step/s]INFO 2026-05-20 18:16:20 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.12 loss:0.007 grdn:0.170 lr:2.5e-06 updt_s:0.089 data_s:0.008
1098
+ Training: 86%|████████████████████████████████▌ | 21440/25000 [37:25<05:54, 10.05step/s]INFO 2026-05-20 18:16:22 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.12 loss:0.007 grdn:0.162 lr:2.5e-06 updt_s:0.088 data_s:0.011
1099
+ Training: 86%|████████████████████████████████▌ | 21460/25000 [37:27<05:58, 9.87step/s]INFO 2026-05-20 18:16:24 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.13 loss:0.007 grdn:0.162 lr:2.5e-06 updt_s:0.090 data_s:0.007
1100
+ Training: 86%|████████████████████████████████▋ | 21479/25000 [37:29<05:41, 10.32step/s]INFO 2026-05-20 18:16:26 ot_train.py:488 step:21K smpl:1M ep:3K epch:5.13 loss:0.007 grdn:0.169 lr:2.4e-06 updt_s:0.090 data_s:0.006
1101
+ Training: 86%|████████████████████████████████▋ | 21500/25000 [37:31<05:22, 10.84step/s]INFO 2026-05-20 18:16:28 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.14 loss:0.008 grdn:0.162 lr:2.4e-06 updt_s:0.090 data_s:0.008
1102
+ Training: 86%|████████████████████████████████▋ | 21520/25000 [37:33<05:58, 9.69step/s]INFO 2026-05-20 18:16:30 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.14 loss:0.007 grdn:0.171 lr:2.4e-06 updt_s:0.092 data_s:0.008
1103
+ Training: 86%|████████████████████████████████▋ | 21540/25000 [37:35<05:55, 9.73step/s]INFO 2026-05-20 18:16:32 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.15 loss:0.007 grdn:0.168 lr:2.4e-06 updt_s:0.087 data_s:0.010
1104
+ Training: 86%|███████��████████████████████████▊ | 21560/25000 [37:37<05:26, 10.53step/s]INFO 2026-05-20 18:16:34 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.15 loss:0.010 grdn:0.187 lr:2.3e-06 updt_s:0.087 data_s:0.008
1105
+ Training: 86%|████████████████████████████████▊ | 21580/25000 [37:39<05:27, 10.45step/s]INFO 2026-05-20 18:16:36 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.16 loss:0.007 grdn:0.171 lr:2.3e-06 updt_s:0.090 data_s:0.008
1106
+ Training: 86%|████████████████████████████████▊ | 21600/25000 [37:41<05:33, 10.19step/s]INFO 2026-05-20 18:16:38 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.16 loss:0.007 grdn:0.179 lr:2.3e-06 updt_s:0.087 data_s:0.008
1107
+ Training: 86%|████████████████████████████████▊ | 21619/25000 [37:43<06:08, 9.18step/s]INFO 2026-05-20 18:16:40 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.16 loss:0.007 grdn:0.161 lr:2.3e-06 updt_s:0.095 data_s:0.006
1108
+ Training: 87%|████████████████████████████████▉ | 21639/25000 [37:45<05:08, 10.89step/s]INFO 2026-05-20 18:16:42 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.17 loss:0.008 grdn:0.189 lr:2.2e-06 updt_s:0.085 data_s:0.009
1109
+ Training: 87%|████████████████████████████████▉ | 21659/25000 [37:47<05:28, 10.17step/s]INFO 2026-05-20 18:16:44 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.17 loss:0.008 grdn:0.168 lr:2.2e-06 updt_s:0.095 data_s:0.007
1110
+ Training: 87%|████████████████████████████████▉ | 21679/25000 [37:49<05:34, 9.93step/s]INFO 2026-05-20 18:16:46 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.18 loss:0.008 grdn:0.180 lr:2.2e-06 updt_s:0.090 data_s:0.009
1111
+ Training: 87%|████████████████████████████████▉ | 21699/25000 [37:51<05:21, 10.27step/s]INFO 2026-05-20 18:16:48 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.18 loss:0.008 grdn:0.185 lr:2.1e-06 updt_s:0.085 data_s:0.013
1112
+ Training: 87%|█████████████████████████████████ | 21720/25000 [37:53<05:12, 10.49step/s]INFO 2026-05-20 18:16:50 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.19 loss:0.008 grdn:0.154 lr:2.1e-06 updt_s:0.087 data_s:0.010
1113
+ Training: 87%|█████████████████████████████████ | 21739/25000 [37:55<05:40, 9.58step/s]INFO 2026-05-20 18:16:52 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.19 loss:0.008 grdn:0.170 lr:2.1e-06 updt_s:0.091 data_s:0.009
1114
+ Training: 87%|█████████████████████████████████ | 21759/25000 [37:57<05:18, 10.18step/s]INFO 2026-05-20 18:16:54 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.20 loss:0.007 grdn:0.167 lr:2.1e-06 updt_s:0.086 data_s:0.010
1115
+ Training: 87%|█████████████████████████████████ | 21779/25000 [37:59<05:22, 9.99step/s]INFO 2026-05-20 18:16:56 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.20 loss:0.008 grdn:0.181 lr:2.0e-06 updt_s:0.091 data_s:0.009
1116
+ Training: 87%|█████████████████████████████████▏ | 21799/25000 [38:01<04:49, 11.06step/s]INFO 2026-05-20 18:16:58 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.21 loss:0.008 grdn:0.181 lr:2.0e-06 updt_s:0.088 data_s:0.006
1117
+ Training: 87%|█████████████████████████████████▏ | 21819/25000 [38:03<05:20, 9.92step/s]INFO 2026-05-20 18:17:00 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.21 loss:0.008 grdn:0.171 lr:2.0e-06 updt_s:0.088 data_s:0.009
1118
+ Training: 87%|█████████████████████████████████▏ | 21840/25000 [38:05<05:16, 9.97step/s]INFO 2026-05-20 18:17:02 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.22 loss:0.008 grdn:0.196 lr:2.0e-06 updt_s:0.092 data_s:0.006
1119
+ Training: 87%|█████████████████████████████████▏ | 21859/25000 [38:07<05:29, 9.54step/s]INFO 2026-05-20 18:17:04 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.22 loss:0.008 grdn:0.169 lr:1.9e-06 updt_s:0.095 data_s:0.008
1120
+ Training: 88%|█████████████████████████████████▎ | 21880/25000 [38:09<04:57, 10.49step/s]INFO 2026-05-20 18:17:06 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.23 loss:0.007 grdn:0.162 lr:1.9e-06 updt_s:0.086 data_s:0.009
1121
+ Training: 88%|█████████████████████████████████▎ | 21900/25000 [38:11<05:25, 9.51step/s]INFO 2026-05-20 18:17:08 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.23 loss:0.008 grdn:0.178 lr:1.9e-06 updt_s:0.093 data_s:0.007
1122
+ Training: 88%|█████████████████████████████████▎ | 21919/25000 [38:13<04:43, 10.88step/s]INFO 2026-05-20 18:17:10 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.24 loss:0.007 grdn:0.169 lr:1.9e-06 updt_s:0.089 data_s:0.006
1123
+ Training: 88%|█████████████████████████████████▎ | 21940/25000 [38:15<05:04, 10.06step/s]INFO 2026-05-20 18:17:12 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.24 loss:0.007 grdn:0.150 lr:1.9e-06 updt_s:0.094 data_s:0.010
1124
+ Training: 88%|█████████████████████████████████▍ | 21959/25000 [38:17<05:18, 9.56step/s]INFO 2026-05-20 18:17:14 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.25 loss:0.009 grdn:0.190 lr:1.8e-06 updt_s:0.089 data_s:0.009
1125
+ Training: 88%|█████████████████████████████████▍ | 21979/25000 [38:19<04:49, 10.44step/s]INFO 2026-05-20 18:17:16 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.25 loss:0.008 grdn:0.182 lr:1.8e-06 updt_s:0.085 data_s:0.010
1126
+ Training: 88%|█████████████████████████████████▍ | 21999/25000 [38:21<04:59, 10.03step/s]INFO 2026-05-20 18:17:18 ot_train.py:488 step:22K smpl:1M ep:3K epch:5.26 loss:0.008 grdn:0.185 lr:1.8e-06 updt_s:0.092 data_s:0.009
1127
+ INFO 2026-05-20 18:17:18 ot_train.py:502 Checkpoint policy after step 22000
1128
+ Traceback (most recent call last):
1129
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/bin/lerobot-train", line 10, in <module>
1130
+ sys.exit(main())
1131
+ ^^^^^^
1132
+ File "/home/shadeform/Desktop/robot_learning/lerobot/src/lerobot/scripts/lerobot_train.py", line 595, in main
1133
+ train()
1134
+ File "/home/shadeform/Desktop/robot_learning/lerobot/src/lerobot/configs/parser.py", line 233, in wrapper_inner
1135
+ response = fn(cfg, *args, **kwargs)
1136
+ ^^^^^^^^^^^^^^^^^^^^^^^^
1137
+ File "/home/shadeform/Desktop/robot_learning/lerobot/src/lerobot/scripts/lerobot_train.py", line 504, in train
1138
+ save_checkpoint(
checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/files/wandb-summary.json ADDED
File without changes
checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/logs/debug-core.log CHANGED
@@ -5,3 +5,11 @@
5
  {"time":"2026-05-20T17:38:54.193646453Z","level":"INFO","msg":"handleInformInit: received","streamId":"ql0pvhal","id":"1(@)"}
6
  {"time":"2026-05-20T17:38:54.405234199Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ql0pvhal","id":"1(@)"}
7
  {"time":"2026-05-20T17:38:59.719739919Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6i6wv3l2mq11"}
 
 
 
 
 
 
 
 
 
5
  {"time":"2026-05-20T17:38:54.193646453Z","level":"INFO","msg":"handleInformInit: received","streamId":"ql0pvhal","id":"1(@)"}
6
  {"time":"2026-05-20T17:38:54.405234199Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ql0pvhal","id":"1(@)"}
7
  {"time":"2026-05-20T17:38:59.719739919Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6i6wv3l2mq11"}
8
+ {"time":"2026-05-20T18:18:46.786764477Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
9
+ {"time":"2026-05-20T18:18:46.7868459Z","level":"INFO","msg":"server is shutting down"}
10
+ {"time":"2026-05-20T18:18:46.786896596Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
11
+ {"time":"2026-05-20T18:18:46.78699247Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
12
+ {"time":"2026-05-20T18:18:46.786985389Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1855373-1855443-2491550268/socket","Net":"unix"}}
13
+ {"time":"2026-05-20T18:18:47.627901131Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
14
+ {"time":"2026-05-20T18:18:47.627931047Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
15
+ {"time":"2026-05-20T18:18:47.627937857Z","level":"INFO","msg":"server is closed"}
checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/logs/debug-internal.log CHANGED
@@ -245,3 +245,82 @@
245
  {"time":"2026-05-20T18:08:24.975880521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
246
  {"time":"2026-05-20T18:08:39.716535976Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":840,"history_lines":8,"events_offset":235,"events_lines":2,"console_offset":865,"console_lines":9}
247
  {"time":"2026-05-20T18:08:39.877448628Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  {"time":"2026-05-20T18:08:24.975880521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
246
  {"time":"2026-05-20T18:08:39.716535976Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":840,"history_lines":8,"events_offset":235,"events_lines":2,"console_offset":865,"console_lines":9}
247
  {"time":"2026-05-20T18:08:39.877448628Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
248
+ {"time":"2026-05-20T18:08:54.716533796Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":848,"history_lines":7,"events_offset":237,"events_lines":2,"console_offset":873,"console_lines":8}
249
+ {"time":"2026-05-20T18:08:54.871911901Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
250
+ {"time":"2026-05-20T18:09:09.717065112Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":855,"history_lines":8,"events_offset":239,"events_lines":2,"console_offset":880,"console_lines":9}
251
+ {"time":"2026-05-20T18:09:09.842775704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
252
+ {"time":"2026-05-20T18:09:24.716992954Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":863,"history_lines":7,"events_offset":241,"events_lines":2,"console_offset":888,"console_lines":8}
253
+ {"time":"2026-05-20T18:09:24.905276893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
254
+ {"time":"2026-05-20T18:09:39.716505143Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":870,"history_lines":8,"events_offset":243,"events_lines":2,"console_offset":895,"console_lines":9}
255
+ {"time":"2026-05-20T18:09:39.927378967Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
256
+ {"time":"2026-05-20T18:09:54.719405323Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":878,"history_lines":7,"events_offset":245,"events_lines":2,"console_offset":903,"console_lines":8}
257
+ {"time":"2026-05-20T18:09:54.962240271Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
258
+ {"time":"2026-05-20T18:10:09.716837911Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":885,"history_lines":8,"events_offset":247,"events_lines":2,"console_offset":910,"console_lines":9}
259
+ {"time":"2026-05-20T18:10:09.923036222Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
260
+ {"time":"2026-05-20T18:10:24.717098427Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":893,"history_lines":6,"events_offset":249,"events_lines":2,"console_offset":918,"console_lines":7}
261
+ {"time":"2026-05-20T18:10:24.849016548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
262
+ {"time":"2026-05-20T18:10:39.716917501Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":899,"history_lines":5,"events_offset":251,"events_lines":2,"console_offset":925,"console_lines":6}
263
+ {"time":"2026-05-20T18:10:39.871365988Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
264
+ {"time":"2026-05-20T18:10:54.717358924Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":904,"history_lines":7,"events_offset":253,"events_lines":2,"console_offset":930,"console_lines":8}
265
+ {"time":"2026-05-20T18:10:54.966989707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
266
+ {"time":"2026-05-20T18:11:09.71665628Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":911,"history_lines":8,"events_offset":255,"events_lines":2,"console_offset":937,"console_lines":9}
267
+ {"time":"2026-05-20T18:11:09.918838619Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
268
+ {"time":"2026-05-20T18:11:24.719326062Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":919,"history_lines":8,"events_offset":257,"events_lines":2,"console_offset":945,"console_lines":9}
269
+ {"time":"2026-05-20T18:11:24.939013016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
270
+ {"time":"2026-05-20T18:11:39.717353513Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":927,"history_lines":7,"events_offset":259,"events_lines":2,"console_offset":953,"console_lines":8}
271
+ {"time":"2026-05-20T18:11:39.947273451Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
272
+ {"time":"2026-05-20T18:11:54.717226684Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":934,"history_lines":8,"events_offset":261,"events_lines":2,"console_offset":960,"console_lines":9}
273
+ {"time":"2026-05-20T18:11:54.9660831Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
274
+ {"time":"2026-05-20T18:12:09.716460033Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":942,"history_lines":7,"events_offset":263,"events_lines":2,"console_offset":968,"console_lines":8}
275
+ {"time":"2026-05-20T18:12:09.928958612Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
276
+ {"time":"2026-05-20T18:12:24.716408262Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":949,"history_lines":8,"events_offset":265,"events_lines":2,"console_offset":975,"console_lines":9}
277
+ {"time":"2026-05-20T18:12:24.857647396Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
278
+ {"time":"2026-05-20T18:12:39.717038918Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":957,"history_lines":7,"events_offset":267,"events_lines":2,"console_offset":983,"console_lines":8}
279
+ {"time":"2026-05-20T18:12:39.854732599Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
280
+ {"time":"2026-05-20T18:12:54.717825093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":964,"history_lines":8,"events_offset":269,"events_lines":2,"console_offset":990,"console_lines":9}
281
+ {"time":"2026-05-20T18:12:54.834004867Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
282
+ {"time":"2026-05-20T18:13:09.716555396Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":972,"history_lines":8,"events_offset":271,"events_lines":2,"console_offset":998,"console_lines":8}
283
+ {"time":"2026-05-20T18:13:09.8229178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
284
+ {"time":"2026-05-20T18:13:24.71717241Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":980,"history_lines":7,"events_offset":273,"events_lines":2,"console_offset":1006,"console_lines":8}
285
+ {"time":"2026-05-20T18:13:24.853483771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
286
+ {"time":"2026-05-20T18:13:39.716643723Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":987,"history_lines":8,"events_offset":275,"events_lines":2,"console_offset":1013,"console_lines":9}
287
+ {"time":"2026-05-20T18:13:39.868988756Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
288
+ {"time":"2026-05-20T18:13:54.716391483Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":995,"history_lines":4,"events_offset":277,"events_lines":2,"console_offset":1021,"console_lines":5}
289
+ {"time":"2026-05-20T18:13:54.833916452Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
290
+ {"time":"2026-05-20T18:14:09.718337714Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":999,"history_lines":6,"events_offset":279,"events_lines":2,"console_offset":1026,"console_lines":7}
291
+ {"time":"2026-05-20T18:14:09.886639112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
292
+ {"time":"2026-05-20T18:14:24.717072023Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1005,"history_lines":8,"events_offset":281,"events_lines":2,"console_offset":1032,"console_lines":9}
293
+ {"time":"2026-05-20T18:14:24.8770512Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
294
+ {"time":"2026-05-20T18:14:39.716644516Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1013,"history_lines":7,"events_offset":283,"events_lines":2,"console_offset":1040,"console_lines":8}
295
+ {"time":"2026-05-20T18:14:39.855086461Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
296
+ {"time":"2026-05-20T18:14:54.716816662Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1020,"history_lines":8,"events_offset":285,"events_lines":2,"console_offset":1047,"console_lines":9}
297
+ {"time":"2026-05-20T18:14:54.854717465Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
298
+ {"time":"2026-05-20T18:15:09.717130685Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1028,"history_lines":6,"events_offset":287,"events_lines":2,"console_offset":1055,"console_lines":7}
299
+ {"time":"2026-05-20T18:15:10.172699697Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
300
+ {"time":"2026-05-20T18:15:24.716991906Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1034,"history_lines":8,"events_offset":289,"events_lines":2,"console_offset":1061,"console_lines":8}
301
+ {"time":"2026-05-20T18:15:24.882878011Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
302
+ {"time":"2026-05-20T18:15:39.716791799Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1042,"history_lines":7,"events_offset":291,"events_lines":2,"console_offset":1069,"console_lines":8}
303
+ {"time":"2026-05-20T18:15:39.870856728Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
304
+ {"time":"2026-05-20T18:15:54.716408817Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1049,"history_lines":8,"events_offset":293,"events_lines":2,"console_offset":1076,"console_lines":8}
305
+ {"time":"2026-05-20T18:15:55.084622379Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
306
+ {"time":"2026-05-20T18:16:09.71677926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1057,"history_lines":7,"events_offset":295,"events_lines":2,"console_offset":1084,"console_lines":8}
307
+ {"time":"2026-05-20T18:16:09.868219122Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
308
+ {"time":"2026-05-20T18:16:24.716458796Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1064,"history_lines":8,"events_offset":297,"events_lines":2,"console_offset":1091,"console_lines":8}
309
+ {"time":"2026-05-20T18:16:24.963282586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
310
+ {"time":"2026-05-20T18:16:39.716595622Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1072,"history_lines":7,"events_offset":299,"events_lines":2,"console_offset":1099,"console_lines":8}
311
+ {"time":"2026-05-20T18:16:39.984253185Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
312
+ {"time":"2026-05-20T18:16:54.717499092Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1079,"history_lines":8,"events_offset":301,"events_lines":2,"console_offset":1106,"console_lines":9}
313
+ {"time":"2026-05-20T18:16:54.870864329Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
314
+ {"time":"2026-05-20T18:17:09.717171832Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1087,"history_lines":7,"events_offset":303,"events_lines":2,"console_offset":1114,"console_lines":8}
315
+ {"time":"2026-05-20T18:17:09.886232883Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
316
+ {"time":"2026-05-20T18:17:24.717170024Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1094,"history_lines":5,"events_offset":305,"events_lines":2,"console_offset":1121,"console_lines":6}
317
+ {"time":"2026-05-20T18:17:24.878279001Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
318
+ {"time":"2026-05-20T18:17:29.717154442Z","level":"ERROR","msg":"failed to write to chunk: failed to append: write /home/shadeform/Desktop/robot_learning/outputs/train/diffusion_dagger_iter1_25000steps_20260520_173844/wandb/run-20260520_173853-ql0pvhal/files/output.log: no space left on device"}
319
+ {"time":"2026-05-20T18:17:39.717020224Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":307,"events_lines":2,"console_offset":1127,"console_lines":22}
320
+ {"time":"2026-05-20T18:17:39.907862923Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
321
+ {"time":"2026-05-20T18:17:54.716618774Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":309,"events_lines":2}
322
+ {"time":"2026-05-20T18:17:54.949353147Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
323
+ {"time":"2026-05-20T18:18:09.717108368Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":311,"events_lines":2}
324
+ {"time":"2026-05-20T18:18:09.928981618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
325
+ {"time":"2026-05-20T18:18:24.716656857Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":313,"events_lines":2}
326
+ {"time":"2026-05-20T18:18:24.856513972Z","level":"INFO","msg":"filestream: request sent","status":
checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/logs/debug.log CHANGED
@@ -17,3 +17,18 @@ config: {'dataset': {'repo_id': 'local/dagger_episodes', 'root': '/home/shadefor
17
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2428] Wrapping output streams.
18
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2451] Redirects installed.
19
  2026-05-20 17:38:54,714 INFO MainThread:1855373 [wandb_init.py:init():1095] run started, returning control to user process
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2428] Wrapping output streams.
18
  2026-05-20 17:38:54,712 INFO MainThread:1855373 [wandb_run.py:_redirect():2451] Redirects installed.
19
  2026-05-20 17:38:54,714 INFO MainThread:1855373 [wandb_init.py:init():1095] run started, returning control to user process
20
+ 2026-05-20 18:18:46,787 INFO wandb-AsyncioManager-main:1855373 [service_client.py:_forward_responses():122] Reached EOF.
21
+ 2026-05-20 18:18:46,787 INFO wandb-AsyncioManager-main:1855373 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles.
22
+ 2026-05-20 18:18:47,226 ERROR wandb-AsyncioManager-main:1855373 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
23
+ Traceback (most recent call last):
24
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
25
+ await fn()
26
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 105, in loop
27
+ await self._poll_and_print(timeout=None)
28
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 133, in _poll_and_print
29
+ handle = await self._interface.deliver_async(request)
30
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
31
+ File "/home/shadeform/Desktop/robot_learning/lerobot/.venv/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
32
+ handle = await self._client.deliver(request)
33
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
34
+ File "/home
checkpoints/dagger/1iter/wandb/run-20260520_173853-ql0pvhal/run-ql0pvhal.wandb CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ced62d248847cb8629a2196cf2042e9f55bb2c785441d01b2893109ea7f0395c
3
- size 2523136
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e8b104e69bf074a58d96c642f72abe90a5eb8578fabb438c67945c38b08bf28
3
+ size 3407872