petkopetkov commited on
Commit
ef261b4
·
verified ·
1 Parent(s): e87b6d5

backup: add missing files to sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1

Browse files
Files changed (19) hide show
  1. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._max_dataframe.__.parquet +3 -0
  2. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._min_dataframe.__.parquet +3 -0
  3. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._rand_dataframe.__.parquet +3 -0
  4. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/config.yaml +3 -215
  5. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/full_config.yaml +3 -496
  6. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/format.log +3 -24
  7. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/model_config.yaml +3 -120
  8. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/language_results.json +3 -0
  9. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/model_config.yaml +3 -137
  10. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/raw_config.yaml +3 -11
  11. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session.log +3 -0
  12. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_info.json +3 -1
  13. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_1.log +3 -526
  14. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_2.log +3 -526
  15. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_3.log +3 -526
  16. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_4.log +3 -526
  17. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_5.log +3 -526
  18. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_6.log +3 -526
  19. sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_7.log +3 -526
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._max_dataframe.__.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0ad9c0c47c0e592e937c931759cbc06d797fcb0316b5bc9ce1873a1ef5c00bd
3
+ size 985
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._min_dataframe.__.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0ad9c0c47c0e592e937c931759cbc06d797fcb0316b5bc9ce1873a1ef5c00bd
3
+ size 985
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._rand_dataframe.__.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0ad9c0c47c0e592e937c931759cbc06d797fcb0316b5bc9ce1873a1ef5c00bd
3
+ size 985
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/config.yaml CHANGED
@@ -1,215 +1,3 @@
1
- session_config:
2
- output_path: /home/petko_petkov/experiments/vlams/control
3
- tag: bridge_curriculum_phase1
4
- metrics_config:
5
- buckets:
6
- wandb:
7
- autoclass: barrel.train.core.trainer.metrics_buckets.wandb_metrics_bucket.WandbMetricsBucket
8
- project: diffusion_vlams
9
- gpus: 8
10
- num_steps: 15000
11
- test_interval: 10000
12
- test_enabled: false
13
- training_module_config:
14
- autoclass: barrel.train.pipes.vlams.pipe.vlam_training_module.VLAMTrainingModule
15
- grad_accumulation_steps: 2
16
- autocast_dtype: bfloat16
17
- fsdp_config:
18
- autoclass: barrel.train.core.common.torch_utils.fsdp.FSDP2
19
- enabled: true
20
- param_dtype: bfloat16
21
- model_config:
22
- autoclass: barrel.train.pipes.vlams.models.vlams.vlam.VLAM
23
- processor_config:
24
- autoclass: barrel.train.pipes.vlams.types.vlam_processors.PiZeroFlowMatchingProcessor
25
- control_io_config:
26
- future_controls_sequence_length: 5
27
- future_controls_sequence_stride_sec: 0.2
28
- rotation_format: quaternion
29
- translation_obs_norm:
30
- autoclass: barrel.train.components.normalization.normalizers.DatasetStatsNormalizer
31
- component_name: translation
32
- mode: bounds_q99
33
- stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
34
- rotation_obs_norm:
35
- autoclass: barrel.train.components.normalization.normalizers.IdentityNormalizer
36
- translation_control_norm:
37
- autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
38
- low:
39
- - -0.04
40
- - -0.04
41
- - -0.04
42
- high:
43
- - 0.04
44
- - 0.04
45
- - 0.04
46
- rotation_control_norm:
47
- autoclass: barrel.train.components.normalization.normalizers.RotationStereomapNormalizer
48
- factor: 4.0
49
- control_tokenizer_config:
50
- autoclass: barrel.train.pipes.vlams.types.action_tokenizer.EmptyTokenizer
51
- num_inference_steps: 10
52
- r0_distribution: uniform
53
- timestep_distribution: beta
54
- distribution_hyperparams:
55
- alpha: 1.5
56
- beta: 1.0
57
- sig_min: 0.001
58
- image_resize: naive
59
- vlm_config:
60
- autoclass: barrel.train.pipes.vlams.models.vlms.paligemma.PaliGemmaVLM
61
- model_id: google/paligemma-3b-mix-224
62
- processor_config:
63
- autoclass: barrel.train.pipes.vlams.types.vlm_processors.PaliGemmaProcessor
64
- image_sizes:
65
- main:
66
- width: 224
67
- height: 224
68
- pretrain_config:
69
- trainable: true
70
- frozen_dtype: bfloat16
71
- state_dict:
72
- checkpoint_path: PRETRAINED_MODULE
73
- control_module_config:
74
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_module.PiZeroFlowMatchingModule
75
- token_size: 1024
76
- rotation_components: 4
77
- noised_control_proj_config:
78
- autoclass: barrel.train.pipes.vlams.models.control.token_proj.NoisedControlProjector
79
- time_embed:
80
- autoclass: barrel.train.components.nn.layers.fourier_features.FourierFeatures
81
- num_features: 1024
82
- layers: []
83
- max_period: 10000.0
84
- layers:
85
- - 8
86
- - 2048
87
- - 1024
88
- - 1024
89
- activation: SiLU
90
- robot_state_proj_config:
91
- autoclass: barrel.train.pipes.vlams.models.control.token_proj.RobotStateProjector
92
- layers:
93
- - 8
94
- - 1024
95
- mode: ee_pose_gripper
96
- activation: SiLU
97
- control_decoder_config:
98
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoder
99
- num_blocks: 18
100
- block_config:
101
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoderBlock
102
- feature_size: 1024
103
- head_dim: 256
104
- num_heads: 8
105
- num_kv_heads: 1
106
- hidden_size: 4096
107
- activation: GELU
108
- activation_kwargs:
109
- approximate: tanh
110
- attn_implementation: sdpa
111
- position_embed_config:
112
- autoclass: barrel.train.components.nn.layers.positional_encodings.RotaryPositionalEncoding
113
- num_embeddings: 512
114
- embedding_dim: 256
115
- base: 10000
116
- loss_config:
117
- autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRoboticsLoss
118
- discount: false
119
- rotation_fm_loss: cosine_similarity
120
- rotation_loss:
121
- autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRotationLoss
122
- method: integrate
123
- step_size_range:
124
- - 0.01
125
- - 1.0
126
- rotation_losses:
127
- geodesic:
128
- autoclass: barrel.train.pipes.vlams.losses.rotation_loss.RotationLoss
129
- loss_format: geodesic
130
- power: 2
131
- llm_loss:
132
- autoclass: barrel.train.pipes.vlams.losses.vlm_loss.VLMLoss
133
- weights:
134
- translation_loss: 1.0
135
- rotation_loss: 1.0
136
- gripper_loss: 1.0
137
- llm_loss: 0.0
138
- waypoints_loss: 1.0
139
- optimizer_config:
140
- optimizer:
141
- autoclass: barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer
142
- torch_optimizer: torch.optim.AdamW
143
- hyperparams:
144
- lr: 5.0e-05
145
- weight_decay: 0.0
146
- scheduler:
147
- autoclass: barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler
148
- schedulers:
149
- warmup:
150
- autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
151
- torch_scheduler: torch.optim.lr_scheduler.LinearLR
152
- hyperparams:
153
- start_factor: 0.0002
154
- end_factor: 1.0
155
- total_iters: 500
156
- const:
157
- autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
158
- torch_scheduler: torch.optim.lr_scheduler.ConstantLR
159
- hyperparams:
160
- factor: 1.0
161
- total_iters: 300000
162
- milestones:
163
- - 500
164
- clip_grad_norm: 1.0
165
- data_module_config:
166
- autoclass: barrel.train.pipes.vlams.pipe.oxe_data_module.OXEDataModule
167
- dataset_configs:
168
- bridge:
169
- autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset.RoboticsDataset
170
- data_source_config:
171
- autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source.RoboticsDataSource
172
- dataset_path: /home/petko_petkov/datasets/oxe/lerobot-video/bridge
173
- stacker_config:
174
- remove_invalid_future_pointers: true
175
- dataset_weights:
176
- bridge: 1.0
177
- batch_size: 1024
178
- callbacks_config:
179
- callbacks:
180
- params_callback:
181
- autoclass: barrel.train.components.callbacks.params_callback.ParametersCallback
182
- progressbar_callback:
183
- autoclass: barrel.train.components.callbacks.progressbar_callback.ProgressbarCallback
184
- print_callback:
185
- autoclass: barrel.train.components.callbacks.trainer_monitor_callback.TrainerMonitorCallback
186
- print_interval: 100
187
- debug_nans_callback:
188
- autoclass: barrel.train.components.callbacks.debug_nans_callback.DebugNansCallback
189
- enabled: true
190
- plot_grads_callback:
191
- autoclass: barrel.train.components.callbacks.plot_grads_callback.PlotGradsCallback
192
- plot_interval: 200
193
- enabled: false
194
- checkpoint_callback:
195
- autoclass: barrel.train.components.callbacks.checkpoint_callback.CheckpointCallback
196
- session_save_interval: 5000
197
- keep_test_checkpoints: false
198
- keep_trainer_steps: []
199
- keep_model_steps:
200
- - 15000
201
- ema_config:
202
- enabled: true
203
- decay: 0.99
204
- interval: 1
205
- start_step: 20000
206
- keep_trainer_percent: []
207
- keep_model_percent: []
208
- visualization_callback:
209
- autoclass: barrel.train.pipes.vlams.pipe.robotics_vis_callback.RoboticsVisualizationCallback
210
- visualization_interval: 100
211
- inference_export_callback:
212
- autoclass: barrel.train.components.callbacks.inference_export_callback.InferenceExportCallback
213
- model_name: pizero_fm_paligemma
214
- test_callback:
215
- autoclass: barrel.train.pipes.vlams.pipe.robotics_test_dataset_callback.RoboticsTestDatasetCallback
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06ea035872d31218c768fdd4506d7f98c7ac60449769a838d5821161c8f61b65
3
+ size 7921
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/full_config.yaml CHANGED
@@ -1,496 +1,3 @@
1
- backend: nccl
2
- callbacks_config:
3
- callbacks:
4
- params_callback:
5
- autoclass: barrel.train.components.callbacks.params_callback.ParametersCallback
6
- enabled: true
7
- progressbar_callback:
8
- autoclass: barrel.train.components.callbacks.progressbar_callback.ProgressbarCallback
9
- enabled: true
10
- print_callback:
11
- autoclass: barrel.train.components.callbacks.trainer_monitor_callback.TrainerMonitorCallback
12
- print_interval: 100
13
- debug_nans_callback:
14
- autoclass: barrel.train.components.callbacks.debug_nans_callback.DebugNansCallback
15
- check_batch: false
16
- check_grads: false
17
- check_loss: true
18
- check_output: false
19
- check_params: false
20
- drop_to_pdb: false
21
- enabled: true
22
- plot_grads_callback:
23
- autoclass: barrel.train.components.callbacks.plot_grads_callback.PlotGradsCallback
24
- enabled: false
25
- names: []
26
- plot_interval: 200
27
- checkpoint_callback:
28
- autoclass: barrel.train.components.callbacks.checkpoint_callback.CheckpointCallback
29
- ema_config:
30
- decay: 0.99
31
- enabled: true
32
- interval: 1
33
- start_step: 20000
34
- enabled: true
35
- keep_model_percent: []
36
- keep_model_steps:
37
- - 15000
38
- keep_test_checkpoints: false
39
- keep_trainer_percent: []
40
- keep_trainer_steps: []
41
- save_at_init: false
42
- separate_model: true
43
- session_save_interval: 5000
44
- visualization_callback:
45
- autoclass: barrel.train.pipes.vlams.pipe.robotics_vis_callback.RoboticsVisualizationCallback
46
- plot_table: false
47
- token_accuracy_metrics: false
48
- visualization_interval: 100
49
- inference_export_callback:
50
- autoclass: barrel.train.components.callbacks.inference_export_callback.InferenceExportCallback
51
- databib_enabled: true
52
- enabled: true
53
- model_name: pizero_fm_paligemma
54
- test_callback:
55
- autoclass: barrel.train.pipes.vlams.pipe.robotics_test_dataset_callback.RoboticsTestDatasetCallback
56
- batch_size: null
57
- dataloader_workers: 2
58
- dataset_configs: {}
59
- enabled: true
60
- max_batches: null
61
- data_module_config:
62
- autoclass: barrel.train.pipes.vlams.pipe.oxe_data_module.OXEDataModule
63
- batch_size: 1024
64
- dataloader_workers: 2
65
- dataset_configs:
66
- bridge:
67
- augment_kwargs:
68
- main:
69
- color_jitter:
70
- brightness:
71
- - 0.9
72
- - 1.1
73
- contrast:
74
- - 0.9
75
- - 1.1
76
- saturation:
77
- - 0.9
78
- - 1.1
79
- hue: 0.05
80
- resize:
81
- scale:
82
- - 0.8
83
- - 1.0
84
- ratio:
85
- - 0.1
86
- - 0.1
87
- wrist:
88
- color_jitter:
89
- brightness:
90
- - 0.9
91
- - 1.1
92
- contrast:
93
- - 0.9
94
- - 1.1
95
- saturation:
96
- - 0.9
97
- - 1.1
98
- hue: 0.05
99
- resize:
100
- scale:
101
- - 0.8
102
- - 1.0
103
- ratio:
104
- - 0.1
105
- - 0.1
106
- autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset.RoboticsDataset
107
- axes_augment_prob: 0.0
108
- balancer_config: null
109
- control_augment_config:
110
- future_controls_augment_window: null
111
- future_frames_augment_window: null
112
- past_frames_augment_window: null
113
- past_scalars_augment_window: null
114
- corrective_augment_prob: 0.0
115
- data_source_config:
116
- autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source.RoboticsDataSource
117
- dataset_path: /home/petko_petkov/datasets/oxe/lerobot-video/bridge
118
- df_transform_config:
119
- drop_columns: []
120
- episode_length_threshold: null
121
- filter_static_ee_frames: true
122
- gripper_threshold: auto
123
- min_episode_displacement_m: 0.05
124
- remap_columns:
125
- observation.images.image_0: observation.images.main_image
126
- observation.images.image_3: observation.images.wrist_image
127
- rotation_threshold_deg: 1.0
128
- transform_kwargs:
129
- augment: false
130
- env_regex: ''
131
- translation_threshold_m: 0.005
132
- skip_no_language: true
133
- split: train[:100%]
134
- dataset_mode: train
135
- num_image_reader_threads: 2
136
- stacker_config:
137
- future_controls_shift_sec: 0.0
138
- future_frames_shift_sec: 0.0
139
- min_future_horizon_sec: null
140
- min_past_horizon_sec: null
141
- past_frames_shift_sec: 0.0
142
- past_scalars_shift_sec: 0.0
143
- remove_invalid_future_pointers: true
144
- remove_invalid_past_pointers: true
145
- row_skip_offset: 0
146
- row_skip_stride: 1
147
- timestamp_max_gap_sec: 0.04
148
- timestamp_precision_sec: 0.005
149
- total_rows: -1
150
- dataset_weights:
151
- bridge: 1.0
152
- debug_mode: false
153
- deterministic: false
154
- gpus: 8
155
- num_steps: 15000
156
- seed: 1222297665
157
- session_config:
158
- config_path: barrel/train/pipes/vlams/configs/control/pizero_fm_paligemma_bridge_curriculum_phase1.yaml
159
- extra_command_line_args: {}
160
- metrics_config:
161
- buckets:
162
- wandb:
163
- autoclass: barrel.train.core.trainer.metrics_buckets.wandb_metrics_bucket.WandbMetricsBucket
164
- enabled: true
165
- fork_from: ''
166
- project: diffusion_vlams
167
- tags: []
168
- enabled: true
169
- tags: []
170
- output_path: /home/petko_petkov/experiments/vlams/control
171
- restore_path: null
172
- session_name: sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
173
- session_path: /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
174
- tag: bridge_curriculum_phase1
175
- test_enabled: false
176
- test_interval: 10000
177
- training_module_config:
178
- autocast_dtype: bfloat16
179
- autocast_enabled: true
180
- autoclass: barrel.train.pipes.vlams.pipe.vlam_training_module.VLAMTrainingModule
181
- fsdp_config:
182
- autoclass: barrel.train.core.common.torch_utils.fsdp.FSDP2
183
- cast_forward_inputs: true
184
- enabled: true
185
- output_dtype: null
186
- param_dtype: bfloat16
187
- reduce_dtype: float32
188
- reshard_after_forward: true
189
- grad_accumulation_steps: 2
190
- loss_config:
191
- autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRoboticsLoss
192
- discount: false
193
- discount_scale_param: 1
194
- discount_shape_param: 1
195
- gripper_fm_loss: mse_loss
196
- llm_loss:
197
- autoclass: barrel.train.pipes.vlams.losses.vlm_loss.VLMLoss
198
- weight: 1.0
199
- weight_ids: null
200
- num_timesteps: null
201
- rotation_fm_loss: cosine_similarity
202
- rotation_loss:
203
- autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRotationLoss
204
- method: integrate
205
- rotation_losses:
206
- geodesic:
207
- autoclass: barrel.train.pipes.vlams.losses.rotation_loss.RotationLoss
208
- loss_format: geodesic
209
- power: 2
210
- rotation_format: ''
211
- step_size_range:
212
- - 0.01
213
- - 1.0
214
- time_delta_sec: null
215
- translation_fm_loss: mse_loss
216
- weights:
217
- translation_loss: 1.0
218
- rotation_loss: 1.0
219
- gripper_loss: 1.0
220
- llm_loss: 0.0
221
- waypoints_loss: 1.0
222
- model_config:
223
- autoclass: barrel.train.pipes.vlams.models.vlams.vlam.VLAM
224
- control_module_config:
225
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_module.PiZeroFlowMatchingModule
226
- control_decoder_config:
227
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoder
228
- block_config:
229
- activation: GELU
230
- activation_kwargs:
231
- approximate: tanh
232
- attn_implementation: sdpa
233
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoderBlock
234
- dropout: 0.0
235
- feature_size: 1024
236
- head_dim: 256
237
- hidden_size: 4096
238
- norm: RMSNorm
239
- num_heads: 8
240
- num_kv_heads: 1
241
- position_embed_config:
242
- autoclass: barrel.train.components.nn.layers.positional_encodings.RotaryPositionalEncoding
243
- base: 10000
244
- cached: true
245
- embedding_dim: 256
246
- num_embeddings: 512
247
- pretrain_config:
248
- exclude_modules: []
249
- fine_tuneable_modules: []
250
- frozen_dtype: float32
251
- state_dict:
252
- checkpoint_path: null
253
- drop_keys: []
254
- hf_filename: null
255
- hf_hub_repo: null
256
- load_key: ''
257
- remap_keys: {}
258
- trainable: null
259
- pretrain_config:
260
- exclude_modules: []
261
- fine_tuneable_modules: []
262
- frozen_dtype: float32
263
- state_dict:
264
- checkpoint_path: null
265
- drop_keys: []
266
- hf_filename: null
267
- hf_hub_repo: null
268
- load_key: ''
269
- remap_keys: {}
270
- trainable: null
271
- num_blocks: 18
272
- pretrain_config:
273
- exclude_modules: []
274
- fine_tuneable_modules: []
275
- frozen_dtype: float32
276
- state_dict:
277
- checkpoint_path: null
278
- drop_keys: []
279
- hf_filename: null
280
- hf_hub_repo: null
281
- load_key: ''
282
- remap_keys: {}
283
- trainable: null
284
- noised_control_proj_config:
285
- activation: SiLU
286
- autoclass: barrel.train.pipes.vlams.models.control.token_proj.NoisedControlProjector
287
- layers:
288
- - 8
289
- - 2048
290
- - 1024
291
- - 1024
292
- norm: null
293
- pretrain_config:
294
- exclude_modules: []
295
- fine_tuneable_modules: []
296
- frozen_dtype: float32
297
- state_dict:
298
- checkpoint_path: null
299
- drop_keys: []
300
- hf_filename: null
301
- hf_hub_repo: null
302
- load_key: ''
303
- remap_keys: {}
304
- trainable: null
305
- time_embed:
306
- activation: SiLU
307
- autoclass: barrel.train.components.nn.layers.fourier_features.FourierFeatures
308
- layers: []
309
- learnable_features: false
310
- max_period: 10000.0
311
- norm: null
312
- num_features: 1024
313
- pretrain_config:
314
- exclude_modules: []
315
- fine_tuneable_modules: []
316
- frozen_dtype: float32
317
- state_dict:
318
- checkpoint_path: null
319
- drop_keys: []
320
- hf_filename: null
321
- hf_hub_repo: null
322
- load_key: ''
323
- remap_keys: {}
324
- trainable: null
325
- pretrain_config:
326
- exclude_modules: []
327
- fine_tuneable_modules: []
328
- frozen_dtype: float32
329
- state_dict:
330
- checkpoint_path: null
331
- drop_keys: []
332
- hf_filename: null
333
- hf_hub_repo: null
334
- load_key: ''
335
- remap_keys: {}
336
- trainable: null
337
- robot_state_proj_config:
338
- activation: SiLU
339
- autoclass: barrel.train.pipes.vlams.models.control.token_proj.RobotStateProjector
340
- fourier: false
341
- layers:
342
- - 8
343
- - 1024
344
- mode: ee_pose_gripper
345
- pretrain_config:
346
- exclude_modules: []
347
- fine_tuneable_modules: []
348
- frozen_dtype: float32
349
- state_dict:
350
- checkpoint_path: null
351
- drop_keys: []
352
- hf_filename: null
353
- hf_hub_repo: null
354
- load_key: ''
355
- remap_keys: {}
356
- trainable: null
357
- rotation_components: 4
358
- token_size: 1024
359
- pretrain_config:
360
- exclude_modules: []
361
- fine_tuneable_modules: []
362
- frozen_dtype: float32
363
- state_dict:
364
- checkpoint_path: null
365
- drop_keys: []
366
- hf_filename: null
367
- hf_hub_repo: null
368
- load_key: ''
369
- remap_keys: {}
370
- trainable: null
371
- processor_config:
372
- autoclass: barrel.train.pipes.vlams.types.vlam_processors.PiZeroFlowMatchingProcessor
373
- control_io_config:
374
- future_control_offset_sec: 0.0
375
- future_controls_sequence_length: 5
376
- future_controls_sequence_stride_sec: 0.2
377
- future_frames_sequence_length: 1
378
- future_frames_sequence_stride_sec: null
379
- past_frames_sequence_length: 1
380
- past_frames_stride_sec: null
381
- past_scalars_sequence_length: 1
382
- past_scalars_stride_sec: null
383
- sequence_frames: 1
384
- sequence_frames_stride_sec: null
385
- control_tokenizer_config:
386
- autoclass: barrel.train.pipes.vlams.types.action_tokenizer.EmptyTokenizer
387
- distribution_hyperparams:
388
- alpha: 1.5
389
- beta: 1.0
390
- image_resize: naive
391
- joints_obs_norm:
392
- autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
393
- high:
394
- - 3.141592653589793
395
- - 3.141592653589793
396
- - 3.141592653589793
397
- - 3.141592653589793
398
- - 3.141592653589793
399
- - 3.141592653589793
400
- - 3.141592653589793
401
- low:
402
- - -3.141592653589793
403
- - -3.141592653589793
404
- - -3.141592653589793
405
- - -3.141592653589793
406
- - -3.141592653589793
407
- - -3.141592653589793
408
- - -3.141592653589793
409
- num_inference_steps: 10
410
- r0_distribution: uniform
411
- rotation_control_frame: eef_delta
412
- rotation_control_norm:
413
- autoclass: barrel.train.components.normalization.normalizers.RotationStereomapNormalizer
414
- factor: 4.0
415
- rotation_format: quaternion
416
- rotation_obs_frame: robot_base
417
- rotation_obs_norm:
418
- autoclass: barrel.train.components.normalization.normalizers.IdentityNormalizer
419
- sig_min: 0.001
420
- timestep_distribution: beta
421
- translation_control_frame: robot_base_delta
422
- translation_control_norm:
423
- autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
424
- high:
425
- - 0.04
426
- - 0.04
427
- - 0.04
428
- low:
429
- - -0.04
430
- - -0.04
431
- - -0.04
432
- translation_obs_frame: robot_base
433
- translation_obs_norm:
434
- autoclass: barrel.train.components.normalization.normalizers.DatasetStatsNormalizer
435
- component_name: translation
436
- mode: bounds_q99
437
- stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
438
- stats_key: ''
439
- vlm_config:
440
- attn_implementation: flash_attention_2
441
- autoclass: barrel.train.pipes.vlams.models.vlms.paligemma.PaliGemmaVLM
442
- depth_tokens: 0
443
- lm_head: false
444
- mean_resizing: false
445
- model_id: google/paligemma-3b-mix-224
446
- paligemma_3d_config: {}
447
- pretrain_config:
448
- exclude_modules: []
449
- fine_tuneable_modules: []
450
- frozen_dtype: bfloat16
451
- state_dict:
452
- checkpoint_path: PRETRAINED_MODULE
453
- drop_keys: []
454
- hf_filename: null
455
- hf_hub_repo: null
456
- load_key: ''
457
- remap_keys: {}
458
- trainable: true
459
- processor_config:
460
- autoclass: barrel.train.pipes.vlams.types.vlm_processors.PaliGemmaProcessor
461
- image_sizes:
462
- main:
463
- height: 224
464
- width: 224
465
- image_token: <image>
466
- max_language_tokens: -1
467
- train_only_depth_tokens: false
468
- optimizer_config:
469
- clip_grad_norm: 1.0
470
- grad_scaler:
471
- enabled: false
472
- optimizer:
473
- autoclass: barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer
474
- hyperparams:
475
- lr: 5.0e-05
476
- weight_decay: 0.0
477
- param_groups: {}
478
- torch_optimizer: torch.optim.AdamW
479
- scheduler:
480
- autoclass: barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler
481
- milestones:
482
- - 500
483
- schedulers:
484
- warmup:
485
- autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
486
- hyperparams:
487
- start_factor: 0.0002
488
- end_factor: 1.0
489
- total_iters: 500
490
- torch_scheduler: torch.optim.lr_scheduler.LinearLR
491
- const:
492
- autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
493
- hyperparams:
494
- factor: 1.0
495
- total_iters: 300000
496
- torch_scheduler: torch.optim.lr_scheduler.ConstantLR
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd5e0b76cb6b7291610aafbeb6924c2760b7ecb4640f84ecb78b3762fe7872be
3
+ size 16020
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/format.log CHANGED
@@ -1,24 +1,3 @@
1
- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m autoflake --in-place --remove-all-unused-imports /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/modeling_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/configuration_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/common_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/processing_pizero_fm_paligemma.py
2
- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m isort /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/modeling_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/configuration_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/common_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/processing_pizero_fm_paligemma.py
3
- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m black --config /home/petko_petkov/barrel/.claude/worktrees/bridge-tread-qwen/pyproject.toml /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/modeling_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/configuration_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/common_pizero_fm_paligemma.py /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/processing_pizero_fm_paligemma.py
4
-
5
-
6
- Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/modeling_pizero_fm_paligemma.py
7
- Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/configuration_pizero_fm_paligemma.py
8
- Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/common_pizero_fm_paligemma.py
9
- Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/processing_pizero_fm_paligemma.py
10
-
11
- reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/configuration_pizero_fm_paligemma.py
12
- reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/common_pizero_fm_paligemma.py
13
- reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/processing_pizero_fm_paligemma.py
14
- reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/modeling_pizero_fm_paligemma.py
15
-
16
- All done! ✨ 🍰 ✨
17
- 4 files reformatted.
18
-
19
-
20
- TO RERUN:
21
-
22
- bazel run //tools:autoflake -- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m autoflake
23
- bazel run //tools:isort -- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m isort
24
- bazel run //tools:black -- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m black
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe0e32cf3f9159943e77c44dc0bae6d89c832b00493874572a1402e909ee9e38
3
+ size 5564
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/model_config.yaml CHANGED
@@ -1,120 +1,3 @@
1
- control_module_config:
2
- control_decoder_config:
3
- block_config:
4
- activation: GELU
5
- activation_kwargs:
6
- approximate: tanh
7
- attn_implementation: sdpa
8
- dropout: 0.0
9
- feature_size: 1024
10
- head_dim: 256
11
- hidden_size: 4096
12
- norm: RMSNorm
13
- num_heads: 8
14
- num_kv_heads: 1
15
- position_embed_config:
16
- base: 10000
17
- cached: true
18
- embedding_dim: 256
19
- num_embeddings: 512
20
- num_blocks: 18
21
- noised_control_proj_config:
22
- activation: SiLU
23
- layers:
24
- - 8
25
- - 2048
26
- - 1024
27
- - 1024
28
- norm: null
29
- time_embed:
30
- activation: SiLU
31
- layers: []
32
- learnable_features: false
33
- max_period: 10000.0
34
- norm: null
35
- num_features: 1024
36
- robot_state_proj_config:
37
- activation: SiLU
38
- fourier: false
39
- layers:
40
- - 8
41
- - 1024
42
- mode: ee_pose_gripper
43
- rotation_components: 4
44
- token_size: 1024
45
- processor_config:
46
- control_io_config:
47
- future_control_offset_sec: 0.0
48
- future_controls_sequence_length: 5
49
- future_controls_sequence_stride_sec: 0.2
50
- future_frames_sequence_length: 1
51
- future_frames_sequence_stride_sec: null
52
- past_frames_sequence_length: 1
53
- past_frames_stride_sec: null
54
- past_scalars_sequence_length: 1
55
- past_scalars_stride_sec: null
56
- sequence_frames: 1
57
- sequence_frames_stride_sec: null
58
- control_tokenizer_config: {}
59
- distribution_hyperparams:
60
- alpha: 1.5
61
- beta: 1.0
62
- image_resize: naive
63
- joints_obs_norm:
64
- high:
65
- - 3.141592653589793
66
- - 3.141592653589793
67
- - 3.141592653589793
68
- - 3.141592653589793
69
- - 3.141592653589793
70
- - 3.141592653589793
71
- - 3.141592653589793
72
- low:
73
- - -3.141592653589793
74
- - -3.141592653589793
75
- - -3.141592653589793
76
- - -3.141592653589793
77
- - -3.141592653589793
78
- - -3.141592653589793
79
- - -3.141592653589793
80
- num_inference_steps: 10
81
- r0_distribution: uniform
82
- rotation_control_frame: eef_delta
83
- rotation_control_norm:
84
- factor: 4.0
85
- rotation_format: quaternion
86
- rotation_obs_frame: robot_base
87
- rotation_obs_norm: {}
88
- sig_min: 0.001
89
- timestep_distribution: beta
90
- translation_control_frame: robot_base_delta
91
- translation_control_norm:
92
- high:
93
- - 0.04
94
- - 0.04
95
- - 0.04
96
- low:
97
- - -0.04
98
- - -0.04
99
- - -0.04
100
- translation_obs_frame: robot_base
101
- translation_obs_norm:
102
- component_name: translation
103
- mode: bounds_q99
104
- stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
105
- stats_key: ''
106
- vlm_config:
107
- attn_implementation: sdpa
108
- depth_tokens: 0
109
- lm_head: false
110
- mean_resizing: false
111
- model_id: google/paligemma-3b-mix-224
112
- paligemma_3d_config: {}
113
- processor_config:
114
- image_sizes:
115
- main:
116
- height: 224
117
- width: 224
118
- image_token: <image>
119
- max_language_tokens: 64
120
- train_only_depth_tokens: false
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f892712e82673a0b76048f0d0fb4a492813c36df8ecbcc54265c66d697dac86
3
+ size 2819
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/language_results.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa48b47d0eb7f4fa77de8d2ab1bb37f2b726e54e20862f9ee6e95e2be09c7745
3
+ size 544466
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/model_config.yaml CHANGED
@@ -1,137 +1,3 @@
1
- autoclass: barrel.train.pipes.vlams.models.vlams.vlam.VLAM
2
- control_module_config:
3
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_module.PiZeroFlowMatchingModule
4
- control_decoder_config:
5
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoder
6
- block_config:
7
- activation: GELU
8
- activation_kwargs:
9
- approximate: tanh
10
- attn_implementation: sdpa
11
- autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoderBlock
12
- dropout: 0.0
13
- feature_size: 1024
14
- head_dim: 256
15
- hidden_size: 4096
16
- norm: RMSNorm
17
- num_heads: 8
18
- num_kv_heads: 1
19
- position_embed_config:
20
- autoclass: barrel.train.components.nn.layers.positional_encodings.RotaryPositionalEncoding
21
- base: 10000
22
- cached: true
23
- embedding_dim: 256
24
- num_embeddings: 512
25
- num_blocks: 18
26
- noised_control_proj_config:
27
- activation: SiLU
28
- autoclass: barrel.train.pipes.vlams.models.control.token_proj.NoisedControlProjector
29
- layers:
30
- - 8
31
- - 2048
32
- - 1024
33
- - 1024
34
- norm: null
35
- time_embed:
36
- activation: SiLU
37
- autoclass: barrel.train.components.nn.layers.fourier_features.FourierFeatures
38
- layers: []
39
- learnable_features: false
40
- max_period: 10000.0
41
- norm: null
42
- num_features: 1024
43
- robot_state_proj_config:
44
- activation: SiLU
45
- autoclass: barrel.train.pipes.vlams.models.control.token_proj.RobotStateProjector
46
- fourier: false
47
- layers:
48
- - 8
49
- - 1024
50
- mode: ee_pose_gripper
51
- rotation_components: 4
52
- token_size: 1024
53
- processor_config:
54
- autoclass: barrel.train.pipes.vlams.types.vlam_processors.PiZeroFlowMatchingProcessor
55
- control_io_config:
56
- future_control_offset_sec: 0.0
57
- future_controls_sequence_length: 5
58
- future_controls_sequence_stride_sec: 0.2
59
- future_frames_sequence_length: 1
60
- future_frames_sequence_stride_sec: null
61
- past_frames_sequence_length: 1
62
- past_frames_stride_sec: null
63
- past_scalars_sequence_length: 1
64
- past_scalars_stride_sec: null
65
- sequence_frames: 1
66
- sequence_frames_stride_sec: null
67
- control_tokenizer_config:
68
- autoclass: barrel.train.pipes.vlams.types.action_tokenizer.EmptyTokenizer
69
- distribution_hyperparams:
70
- alpha: 1.5
71
- beta: 1.0
72
- image_resize: naive
73
- joints_obs_norm:
74
- autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
75
- high:
76
- - 3.141592653589793
77
- - 3.141592653589793
78
- - 3.141592653589793
79
- - 3.141592653589793
80
- - 3.141592653589793
81
- - 3.141592653589793
82
- - 3.141592653589793
83
- low:
84
- - -3.141592653589793
85
- - -3.141592653589793
86
- - -3.141592653589793
87
- - -3.141592653589793
88
- - -3.141592653589793
89
- - -3.141592653589793
90
- - -3.141592653589793
91
- num_inference_steps: 10
92
- r0_distribution: uniform
93
- rotation_control_frame: eef_delta
94
- rotation_control_norm:
95
- autoclass: barrel.train.components.normalization.normalizers.RotationStereomapNormalizer
96
- factor: 4.0
97
- rotation_format: quaternion
98
- rotation_obs_frame: robot_base
99
- rotation_obs_norm:
100
- autoclass: barrel.train.components.normalization.normalizers.IdentityNormalizer
101
- sig_min: 0.001
102
- timestep_distribution: beta
103
- translation_control_frame: robot_base_delta
104
- translation_control_norm:
105
- autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
106
- high:
107
- - 0.04
108
- - 0.04
109
- - 0.04
110
- low:
111
- - -0.04
112
- - -0.04
113
- - -0.04
114
- translation_obs_frame: robot_base
115
- translation_obs_norm:
116
- autoclass: barrel.train.components.normalization.normalizers.DatasetStatsNormalizer
117
- component_name: translation
118
- mode: bounds_q99
119
- stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
120
- stats_key: ''
121
- vlm_config:
122
- attn_implementation: flash_attention_2
123
- autoclass: barrel.train.pipes.vlams.models.vlms.paligemma.PaliGemmaVLM
124
- depth_tokens: 0
125
- lm_head: false
126
- mean_resizing: false
127
- model_id: google/paligemma-3b-mix-224
128
- paligemma_3d_config: {}
129
- processor_config:
130
- autoclass: barrel.train.pipes.vlams.types.vlm_processors.PaliGemmaProcessor
131
- image_sizes:
132
- main:
133
- height: 224
134
- width: 224
135
- image_token: <image>
136
- max_language_tokens: -1
137
- train_only_depth_tokens: false
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfb3f9546044efb942cefc95d211e03d5cda1aeb8d25bd9f29a7fe69ff5bb792
3
+ size 4278
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/raw_config.yaml CHANGED
@@ -1,11 +1,3 @@
1
- import: pizero_fm_paligemma_bridge.yaml
2
-
3
- session_config:
4
- tag: bridge_curriculum_phase1
5
-
6
- num_steps: &num_steps 15000
7
-
8
- callbacks_config:
9
- callbacks:
10
- checkpoint_callback:
11
- keep_model_steps: !overwrite [15000]
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0b90ff29d4b11cf45658aa01d93964b33108ab0eb7c5915001e05f896ff1435
3
+ size 218
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session.log ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f24870cf60647678b0e12eb39f8c8ef37eab3922c28235fff5497dbe23ad6e5d
3
+ size 18986089
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_info.json CHANGED
@@ -1 +1,3 @@
1
- {"batch_size": 1024, "command": "/scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/components/scripts/train/train.py --config barrel/train/pipes/vlams/configs/control/pizero_fm_paligemma_bridge_curriculum_phase1.yaml", "docker_image": "no-docker-image-detected", "git_branch": "worktree-bridge-tread-qwen", "git_hash": "0d09dc2c2ac847058dcacf1bf838091557f3e380", "gpu_type": "NVIDIA H200", "hostname": "msp3-2", "launched": "Sunday, 05 April 2026 at 07:47:24 UTC", "num_gpus": 8, "num_nodes": 1, "num_steps": 15000, "platform": "slurm", "session_name": "sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1", "session_path": "/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1", "username": "petko_petkov"}
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df0ab680e7e76361f4cd0df6923b4fb35a011220a468dd1ba1a01d83205e7c1b
3
+ size 936
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_1.log CHANGED
@@ -1,526 +1,3 @@
1
- INFO 2026-04-05 07:47:39,913 barrel.train.components.optimizers.optimizer_module init_optimizer rank=1, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
2
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
3
- hyperparams=ConfigDict[typing.Any](
4
- lr=5e-05,
5
- weight_decay=0.0,
6
- ),
7
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
8
- torch_optimizer='torch.optim.AdamW',
9
- ), scheduler=SequentialSchedulerConfig(
10
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
11
- milestones=ConfigList[int](
12
- 500,
13
- ),
14
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
15
- warmup=TorchSchedulerConfig(
16
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
17
- hyperparams=ConfigDict[typing.Any](
18
- start_factor=0.0002,
19
- end_factor=1.0,
20
- total_iters=500,
21
- ),
22
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
23
- ),
24
- const=TorchSchedulerConfig(
25
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
26
- hyperparams=ConfigDict[typing.Any](
27
- factor=1.0,
28
- total_iters=300000,
29
- ),
30
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
31
- ),
32
- ),
33
- )
34
- INFO 2026-04-05 07:47:39,914 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
35
- INFO 2026-04-05 07:47:41,966 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge
36
- INFO 2026-04-05 07:48:06,548 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
37
- INFO 2026-04-05 07:48:10,913 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
38
- INFO 2026-04-05 07:48:10,913 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=28947.0, dataset_name=bridge
39
- INFO 2026-04-05 07:48:10,953 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183254, rank=1
40
- INFO 2026-04-05 07:48:11,805 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge rank=1, status=start
41
- INFO 2026-04-05 07:48:11,844 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
42
- INFO 2026-04-05 07:48:11,846 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=31931.7
43
- INFO 2026-04-05 07:48:34,458 barrel.train.core.trainer.trainer session_init status=success, rank=1
44
- INFO 2026-04-05 07:48:34,460 barrel.train.core.trainer.trainer start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
45
- INFO 2026-04-05 07:48:34,519 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-2, worker_pid=3875752, parent_pid=3872890, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
46
- INFO 2026-04-05 07:48:34,552 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-2, worker_pid=3875770, parent_pid=3872890, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
47
- INFO 2026-04-05 07:51:52,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
48
- INFO 2026-04-05 07:54:44,997 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
49
- INFO 2026-04-05 07:57:35,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
50
- INFO 2026-04-05 08:00:26,862 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
51
- INFO 2026-04-05 08:03:18,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
52
- INFO 2026-04-05 08:06:09,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
53
- INFO 2026-04-05 08:09:01,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
54
- INFO 2026-04-05 08:11:53,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
55
- INFO 2026-04-05 08:14:44,578 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
56
- INFO 2026-04-05 08:17:37,375 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
57
- INFO 2026-04-05 08:20:30,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
58
- INFO 2026-04-05 08:23:23,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
59
- INFO 2026-04-05 08:26:15,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
60
- INFO 2026-04-05 08:29:07,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
61
- INFO 2026-04-05 08:31:59,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
62
- INFO 2026-04-05 08:34:50,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
63
- INFO 2026-04-05 08:37:43,456 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
64
- INFO 2026-04-05 08:40:35,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
65
- INFO 2026-04-05 08:43:28,467 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
66
- INFO 2026-04-05 08:46:21,132 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
67
- INFO 2026-04-05 08:49:13,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
68
- INFO 2026-04-05 08:52:04,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
69
- INFO 2026-04-05 08:54:56,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
70
- INFO 2026-04-05 08:57:48,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
71
- INFO 2026-04-05 09:00:40,238 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
72
- INFO 2026-04-05 09:03:32,325 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
73
- INFO 2026-04-05 09:06:23,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
74
- INFO 2026-04-05 09:09:15,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
75
- INFO 2026-04-05 09:12:08,117 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
76
- INFO 2026-04-05 09:14:59,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
77
- INFO 2026-04-05 09:17:51,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
78
- INFO 2026-04-05 09:20:42,298 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
79
- INFO 2026-04-05 09:23:34,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
80
- INFO 2026-04-05 09:26:26,064 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
81
- INFO 2026-04-05 09:29:18,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
82
- INFO 2026-04-05 09:32:09,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
83
- INFO 2026-04-05 09:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
84
- INFO 2026-04-05 09:37:52,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
85
- INFO 2026-04-05 09:40:43,428 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
86
- INFO 2026-04-05 09:43:34,586 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
87
- INFO 2026-04-05 09:46:26,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
88
- INFO 2026-04-05 09:49:17,267 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
89
- INFO 2026-04-05 09:52:06,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
90
- INFO 2026-04-05 09:54:57,889 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
91
- INFO 2026-04-05 09:57:48,894 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
92
- INFO 2026-04-05 10:00:41,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
93
- INFO 2026-04-05 10:03:33,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
94
- INFO 2026-04-05 10:06:24,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
95
- INFO 2026-04-05 10:09:17,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
96
- INFO 2026-04-05 10:12:18,772 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
97
- INFO 2026-04-05 10:15:33,483 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
98
- INFO 2026-04-05 10:18:24,744 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
99
- INFO 2026-04-05 10:21:17,303 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
100
- INFO 2026-04-05 10:24:08,917 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
101
- INFO 2026-04-05 10:27:00,005 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
102
- INFO 2026-04-05 10:29:51,959 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
103
- INFO 2026-04-05 10:32:42,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
104
- INFO 2026-04-05 10:35:34,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
105
- INFO 2026-04-05 10:38:25,583 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
106
- INFO 2026-04-05 10:41:18,350 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
107
- INFO 2026-04-05 10:44:08,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
108
- INFO 2026-04-05 10:47:00,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
109
- INFO 2026-04-05 10:49:51,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
110
- INFO 2026-04-05 10:52:43,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
111
- INFO 2026-04-05 10:55:35,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
112
- INFO 2026-04-05 10:58:26,949 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
113
- INFO 2026-04-05 11:01:18,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
114
- INFO 2026-04-05 11:04:10,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
115
- INFO 2026-04-05 11:07:02,461 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
116
- INFO 2026-04-05 11:09:53,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
117
- INFO 2026-04-05 11:12:45,451 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
118
- INFO 2026-04-05 11:15:36,492 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
119
- INFO 2026-04-05 11:18:28,400 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
120
- INFO 2026-04-05 11:21:20,007 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
121
- INFO 2026-04-05 11:24:11,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
122
- INFO 2026-04-05 11:27:02,198 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
123
- INFO 2026-04-05 11:29:53,736 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
124
- INFO 2026-04-05 11:32:44,011 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
125
- INFO 2026-04-05 11:35:35,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
126
- INFO 2026-04-05 11:38:25,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
127
- INFO 2026-04-05 11:41:15,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
128
- INFO 2026-04-05 11:44:05,134 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
129
- INFO 2026-04-05 11:46:56,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
130
- INFO 2026-04-05 11:49:47,469 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
131
- INFO 2026-04-05 11:52:38,563 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
132
- INFO 2026-04-05 11:55:30,290 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
133
- INFO 2026-04-05 11:58:21,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
134
- INFO 2026-04-05 12:01:13,424 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
135
- INFO 2026-04-05 12:04:04,015 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
136
- INFO 2026-04-05 12:06:55,196 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
137
- INFO 2026-04-05 12:09:46,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
138
- INFO 2026-04-05 12:12:37,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
139
- INFO 2026-04-05 12:15:28,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
140
- INFO 2026-04-05 12:18:19,903 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
141
- INFO 2026-04-05 12:21:11,565 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
142
- INFO 2026-04-05 12:24:02,399 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
143
- INFO 2026-04-05 12:26:54,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
144
- INFO 2026-04-05 12:29:45,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
145
- INFO 2026-04-05 12:32:36,286 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
146
- INFO 2026-04-05 12:35:36,863 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
147
- INFO 2026-04-05 12:38:50,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
148
- INFO 2026-04-05 12:41:41,343 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
149
- INFO 2026-04-05 12:44:32,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
150
- INFO 2026-04-05 12:47:23,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
151
- INFO 2026-04-05 12:50:13,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
152
- INFO 2026-04-05 12:53:06,274 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
153
- INFO 2026-04-05 12:55:56,980 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
154
- INFO 2026-04-05 12:58:48,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
155
- INFO 2026-04-05 13:01:39,887 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
156
- INFO 2026-04-05 13:04:31,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
157
- INFO 2026-04-05 13:07:22,319 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
158
- INFO 2026-04-05 13:10:15,398 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
159
- INFO 2026-04-05 13:13:06,519 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
160
- INFO 2026-04-05 13:15:57,164 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
161
- INFO 2026-04-05 13:18:49,647 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
162
- INFO 2026-04-05 13:21:40,162 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
163
- INFO 2026-04-05 13:24:31,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
164
- INFO 2026-04-05 13:27:22,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
165
- INFO 2026-04-05 13:30:13,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
166
- INFO 2026-04-05 13:33:06,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
167
- INFO 2026-04-05 13:35:57,606 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
168
- INFO 2026-04-05 13:38:47,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
169
- INFO 2026-04-05 13:41:40,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
170
- INFO 2026-04-05 13:44:31,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
171
- INFO 2026-04-05 13:47:23,254 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
172
- INFO 2026-04-05 13:50:14,660 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
173
- INFO 2026-04-05 13:53:06,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
174
- INFO 2026-04-05 13:55:58,107 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
175
- INFO 2026-04-05 13:58:50,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
176
- INFO 2026-04-05 14:01:41,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
177
- INFO 2026-04-05 14:04:32,278 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
178
- INFO 2026-04-05 14:07:24,300 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
179
- INFO 2026-04-05 14:10:15,785 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
180
- INFO 2026-04-05 14:13:07,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
181
- INFO 2026-04-05 14:15:58,762 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
182
- INFO 2026-04-05 14:18:49,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
183
- INFO 2026-04-05 14:21:40,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
184
- INFO 2026-04-05 14:24:31,804 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
185
- INFO 2026-04-05 14:27:22,607 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
186
- INFO 2026-04-05 14:30:14,379 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
187
- INFO 2026-04-05 14:33:06,049 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
188
- INFO 2026-04-05 14:35:56,094 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
189
- INFO 2026-04-05 14:38:48,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
190
- INFO 2026-04-05 14:41:40,352 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
191
- INFO 2026-04-05 14:44:32,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
192
- INFO 2026-04-05 14:47:25,215 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
193
- INFO 2026-04-05 14:50:15,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
194
- INFO 2026-04-05 14:53:07,347 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
195
- INFO 2026-04-05 14:56:00,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
196
- INFO 2026-04-05 14:59:02,313 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
197
- INFO 2026-04-05 14:59:35,376 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=1
198
- INFO 2026-04-06 07:33:53,247 barrel.train.components.optimizers.optimizer_module init_optimizer rank=1, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
199
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
200
- hyperparams=ConfigDict[typing.Any](
201
- lr=5e-05,
202
- weight_decay=0.0,
203
- ),
204
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
205
- torch_optimizer='torch.optim.AdamW',
206
- ), scheduler=SequentialSchedulerConfig(
207
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
208
- milestones=ConfigList[int](
209
- 500,
210
- ),
211
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
212
- warmup=TorchSchedulerConfig(
213
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
214
- hyperparams=ConfigDict[typing.Any](
215
- start_factor=0.0002,
216
- end_factor=1.0,
217
- total_iters=500,
218
- ),
219
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
220
- ),
221
- const=TorchSchedulerConfig(
222
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
223
- hyperparams=ConfigDict[typing.Any](
224
- factor=1.0,
225
- total_iters=300000,
226
- ),
227
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
228
- ),
229
- ),
230
- )
231
- INFO 2026-04-06 07:33:53,250 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
232
- INFO 2026-04-06 07:33:56,251 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
233
- INFO 2026-04-06 07:34:13,988 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
234
- INFO 2026-04-06 07:34:19,398 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
235
- INFO 2026-04-06 07:34:23,946 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
236
- INFO 2026-04-06 07:34:23,948 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=27697.1, dataset_name=bridge_paraphrase_k5
237
- INFO 2026-04-06 07:34:23,993 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183254, rank=1
238
- INFO 2026-04-06 07:34:24,858 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=1, status=start
239
- INFO 2026-04-06 07:34:24,906 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
240
- INFO 2026-04-06 07:34:24,910 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=31660.3
241
- INFO 2026-04-06 07:34:24,918 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=1, status=start
242
- INFO 2026-04-06 07:36:05,815 barrel.train.core.trainer.trainer Restore session checkpoint rank=1, status=success
243
- INFO 2026-04-06 07:36:27,875 barrel.train.core.trainer.trainer session_init status=success, rank=1
244
- INFO 2026-04-06 07:36:27,879 barrel.train.core.trainer.trainer start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
245
- INFO 2026-04-06 07:36:27,937 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=2465169, parent_pid=2461886, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
246
- INFO 2026-04-06 07:36:27,977 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=2465195, parent_pid=2461886, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
247
- INFO 2026-04-06 07:39:43,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
248
- INFO 2026-04-06 07:42:35,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
249
- INFO 2026-04-06 07:45:26,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
250
- INFO 2026-04-06 07:48:18,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
251
- INFO 2026-04-06 07:51:08,587 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
252
- INFO 2026-04-06 07:54:00,463 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
253
- INFO 2026-04-06 07:56:51,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
254
- INFO 2026-04-06 07:59:43,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
255
- INFO 2026-04-06 08:02:34,656 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
256
- INFO 2026-04-06 08:05:26,547 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
257
- INFO 2026-04-06 08:08:18,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
258
- INFO 2026-04-06 08:11:11,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
259
- INFO 2026-04-06 08:14:03,148 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
260
- INFO 2026-04-06 08:16:57,119 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
261
- INFO 2026-04-06 08:19:48,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
262
- INFO 2026-04-06 08:22:39,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
263
- INFO 2026-04-06 08:25:31,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
264
- INFO 2026-04-06 08:28:25,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
265
- INFO 2026-04-06 08:31:16,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
266
- INFO 2026-04-06 08:34:09,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
267
- INFO 2026-04-06 08:37:01,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
268
- INFO 2026-04-06 08:39:53,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
269
- INFO 2026-04-06 08:42:45,381 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
270
- INFO 2026-04-06 08:45:38,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
271
- INFO 2026-04-06 08:48:30,989 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
272
- INFO 2026-04-06 08:51:23,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
273
- INFO 2026-04-06 08:54:13,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
274
- INFO 2026-04-06 08:57:05,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
275
- INFO 2026-04-06 08:59:57,948 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
276
- INFO 2026-04-06 09:02:49,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
277
- INFO 2026-04-06 09:05:41,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
278
- INFO 2026-04-06 09:08:33,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
279
- INFO 2026-04-06 09:11:25,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
280
- INFO 2026-04-06 09:14:17,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
281
- INFO 2026-04-06 09:17:08,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
282
- INFO 2026-04-06 09:20:01,791 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
283
- INFO 2026-04-06 09:22:53,270 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
284
- INFO 2026-04-06 09:25:46,358 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
285
- INFO 2026-04-06 09:28:40,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
286
- INFO 2026-04-06 09:31:33,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
287
- INFO 2026-04-06 09:34:24,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
288
- INFO 2026-04-06 09:37:17,501 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
289
- INFO 2026-04-06 09:40:08,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
290
- INFO 2026-04-06 09:43:00,699 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
291
- INFO 2026-04-06 09:45:52,659 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
292
- INFO 2026-04-06 09:48:44,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
293
- INFO 2026-04-06 09:51:36,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
294
- INFO 2026-04-06 09:54:29,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
295
- INFO 2026-04-06 09:57:20,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
296
- INFO 2026-04-06 10:00:22,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
297
- INFO 2026-04-06 10:03:48,882 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
298
- INFO 2026-04-06 10:06:52,075 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
299
- INFO 2026-04-06 10:09:59,025 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
300
- INFO 2026-04-06 10:13:05,837 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
301
- INFO 2026-04-06 10:16:11,930 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
302
- INFO 2026-04-06 10:19:15,973 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
303
- INFO 2026-04-06 10:22:24,656 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
304
- INFO 2026-04-06 10:25:33,535 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
305
- INFO 2026-04-06 10:28:42,180 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
306
- INFO 2026-04-06 10:31:51,507 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
307
- INFO 2026-04-06 10:35:00,437 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
308
- INFO 2026-04-06 10:38:08,598 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
309
- INFO 2026-04-06 10:41:15,590 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
310
- INFO 2026-04-06 11:29:31,531 barrel.train.components.optimizers.optimizer_module init_optimizer rank=1, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
311
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
312
- hyperparams=ConfigDict[typing.Any](
313
- lr=5e-05,
314
- weight_decay=0.0,
315
- ),
316
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
317
- torch_optimizer='torch.optim.AdamW',
318
- ), scheduler=SequentialSchedulerConfig(
319
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
320
- milestones=ConfigList[int](
321
- 500,
322
- ),
323
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
324
- warmup=TorchSchedulerConfig(
325
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
326
- hyperparams=ConfigDict[typing.Any](
327
- start_factor=0.0002,
328
- end_factor=1.0,
329
- total_iters=500,
330
- ),
331
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
332
- ),
333
- const=TorchSchedulerConfig(
334
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
335
- hyperparams=ConfigDict[typing.Any](
336
- factor=1.0,
337
- total_iters=300000,
338
- ),
339
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
340
- ),
341
- ),
342
- )
343
- INFO 2026-04-06 11:29:31,533 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
344
- INFO 2026-04-06 11:29:34,907 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
345
- INFO 2026-04-06 11:29:43,556 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
346
- INFO 2026-04-06 11:29:47,858 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
347
- INFO 2026-04-06 11:29:51,526 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
348
- INFO 2026-04-06 11:29:51,527 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=16619.9, dataset_name=bridge_paraphrase_k5
349
- INFO 2026-04-06 11:29:51,568 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183254, rank=1
350
- INFO 2026-04-06 11:29:52,392 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=1, status=start
351
- INFO 2026-04-06 11:29:52,431 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
352
- INFO 2026-04-06 11:29:52,433 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=20899.3
353
- INFO 2026-04-06 11:29:52,439 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=1, status=start
354
- INFO 2026-04-06 11:31:39,839 barrel.train.core.trainer.trainer Restore session checkpoint rank=1, status=success
355
- INFO 2026-04-06 11:32:01,842 barrel.train.core.trainer.trainer session_init status=success, rank=1
356
- INFO 2026-04-06 11:32:01,844 barrel.train.core.trainer.trainer start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
357
- INFO 2026-04-06 11:32:02,009 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-7, worker_pid=3492780, parent_pid=3489441, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
358
- INFO 2026-04-06 11:32:02,044 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-7, worker_pid=3492815, parent_pid=3489441, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
359
- INFO 2026-04-06 11:35:29,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
360
- INFO 2026-04-06 11:38:35,205 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
361
- INFO 2026-04-06 11:41:41,427 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
362
- INFO 2026-04-06 11:44:46,848 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
363
- INFO 2026-04-06 11:47:52,316 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
364
- INFO 2026-04-06 11:50:56,954 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
365
- INFO 2026-04-06 11:54:03,028 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
366
- INFO 2026-04-06 11:57:07,422 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
367
- INFO 2026-04-06 12:00:13,665 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
368
- INFO 2026-04-06 12:03:19,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
369
- INFO 2026-04-06 12:06:23,694 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
370
- INFO 2026-04-06 12:09:29,445 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
371
- INFO 2026-04-06 12:12:35,570 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
372
- INFO 2026-04-06 12:15:41,417 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
373
- INFO 2026-04-06 12:18:46,704 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
374
- INFO 2026-04-06 12:21:51,861 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
375
- INFO 2026-04-06 12:24:57,747 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
376
- INFO 2026-04-06 12:28:01,899 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
377
- INFO 2026-04-06 12:31:07,224 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
378
- INFO 2026-04-06 12:34:11,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
379
- INFO 2026-04-06 12:37:16,740 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
380
- INFO 2026-04-06 12:40:21,216 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
381
- INFO 2026-04-06 12:43:25,432 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
382
- INFO 2026-04-06 12:46:31,795 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
383
- INFO 2026-04-06 12:49:37,508 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
384
- INFO 2026-04-06 12:52:43,064 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
385
- INFO 2026-04-06 12:55:47,771 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
386
- INFO 2026-04-06 12:58:53,489 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
387
- INFO 2026-04-06 13:01:58,056 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
388
- INFO 2026-04-06 13:05:03,713 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
389
- INFO 2026-04-06 13:08:09,161 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
390
- INFO 2026-04-06 13:11:13,195 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
391
- INFO 2026-04-06 13:14:19,368 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
392
- INFO 2026-04-06 13:17:25,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
393
- INFO 2026-04-06 13:20:30,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
394
- INFO 2026-04-06 13:23:35,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
395
- INFO 2026-04-06 13:26:41,882 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
396
- INFO 2026-04-06 13:29:48,081 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
397
- INFO 2026-04-06 13:32:52,616 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
398
- INFO 2026-04-06 13:35:57,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
399
- INFO 2026-04-06 13:39:03,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
400
- INFO 2026-04-06 13:42:10,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
401
- INFO 2026-04-06 13:45:14,630 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
402
- INFO 2026-04-06 13:48:20,969 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
403
- INFO 2026-04-06 13:51:25,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
404
- INFO 2026-04-06 13:54:30,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
405
- INFO 2026-04-06 13:57:36,003 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
406
- INFO 2026-04-06 14:00:41,876 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
407
- INFO 2026-04-06 14:03:49,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
408
- INFO 2026-04-06 14:07:04,774 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
409
- INFO 2026-04-06 14:10:39,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
410
- INFO 2026-04-06 14:13:44,085 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
411
- INFO 2026-04-06 14:16:49,283 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
412
- INFO 2026-04-06 14:19:53,528 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
413
- INFO 2026-04-06 14:22:58,782 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
414
- INFO 2026-04-06 14:26:03,760 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
415
- INFO 2026-04-06 14:29:07,809 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
416
- INFO 2026-04-06 14:32:11,851 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
417
- INFO 2026-04-06 14:35:16,305 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
418
- INFO 2026-04-06 14:38:22,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
419
- INFO 2026-04-06 14:41:25,472 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
420
- INFO 2026-04-06 14:44:30,525 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
421
- INFO 2026-04-06 14:47:36,050 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
422
- INFO 2026-04-06 14:50:40,765 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
423
- INFO 2026-04-06 14:53:45,035 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
424
- INFO 2026-04-06 14:56:50,252 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
425
- INFO 2026-04-06 14:59:56,683 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
426
- INFO 2026-04-06 15:03:02,324 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
427
- INFO 2026-04-07 00:41:05,378 barrel.train.components.optimizers.optimizer_module init_optimizer rank=1, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
428
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
429
- hyperparams=ConfigDict[typing.Any](
430
- lr=5e-05,
431
- weight_decay=0.0,
432
- ),
433
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
434
- torch_optimizer='torch.optim.AdamW',
435
- ), scheduler=SequentialSchedulerConfig(
436
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
437
- milestones=ConfigList[int](
438
- 500,
439
- ),
440
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
441
- warmup=TorchSchedulerConfig(
442
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
443
- hyperparams=ConfigDict[typing.Any](
444
- start_factor=0.0002,
445
- end_factor=1.0,
446
- total_iters=500,
447
- ),
448
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
449
- ),
450
- const=TorchSchedulerConfig(
451
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
452
- hyperparams=ConfigDict[typing.Any](
453
- factor=1.0,
454
- total_iters=300000,
455
- ),
456
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
457
- ),
458
- ),
459
- )
460
- INFO 2026-04-07 00:41:05,381 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
461
- INFO 2026-04-07 00:41:08,565 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
462
- INFO 2026-04-07 00:41:17,583 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
463
- INFO 2026-04-07 00:41:22,441 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
464
- INFO 2026-04-07 00:41:26,525 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
465
- INFO 2026-04-07 00:41:26,526 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=17960.8, dataset_name=bridge_paraphrase_k5
466
- INFO 2026-04-07 00:41:26,565 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183254, rank=1
467
- INFO 2026-04-07 00:41:27,562 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=1, status=start
468
- INFO 2026-04-07 00:41:27,601 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
469
- INFO 2026-04-07 00:41:27,602 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=22221.5
470
- INFO 2026-04-07 00:41:27,607 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=1, status=start
471
- INFO 2026-04-07 00:43:48,421 barrel.train.core.trainer.trainer Restore session checkpoint rank=1, status=success
472
- INFO 2026-04-07 00:44:10,421 barrel.train.core.trainer.trainer session_init status=success, rank=1
473
- INFO 2026-04-07 00:44:10,423 barrel.train.core.trainer.trainer start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
474
- INFO 2026-04-07 00:44:10,488 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=3313510, parent_pid=3310179, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
475
- INFO 2026-04-07 00:44:10,498 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=3313518, parent_pid=3310179, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
476
- INFO 2026-04-07 00:47:36,029 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
477
- INFO 2026-04-07 00:50:38,869 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
478
- INFO 2026-04-07 00:53:42,313 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
479
- INFO 2026-04-07 00:56:45,086 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
480
- INFO 2026-04-07 00:59:47,986 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
481
- INFO 2026-04-07 01:02:50,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
482
- INFO 2026-04-07 01:05:53,527 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
483
- INFO 2026-04-07 01:08:56,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
484
- INFO 2026-04-07 01:11:58,574 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
485
- INFO 2026-04-07 01:15:02,202 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
486
- INFO 2026-04-07 01:18:05,103 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
487
- INFO 2026-04-07 01:21:08,473 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
488
- INFO 2026-04-07 01:24:11,699 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
489
- INFO 2026-04-07 01:27:15,143 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
490
- INFO 2026-04-07 01:30:16,884 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
491
- INFO 2026-04-07 01:33:19,435 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
492
- INFO 2026-04-07 01:36:22,455 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
493
- INFO 2026-04-07 01:39:25,142 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
494
- INFO 2026-04-07 01:42:29,620 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
495
- INFO 2026-04-07 01:45:32,634 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
496
- INFO 2026-04-07 01:48:36,837 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
497
- INFO 2026-04-07 01:51:39,490 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
498
- INFO 2026-04-07 01:54:42,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
499
- INFO 2026-04-07 01:57:46,410 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
500
- INFO 2026-04-07 02:00:50,170 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
501
- INFO 2026-04-07 02:03:54,279 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
502
- INFO 2026-04-07 02:06:57,975 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
503
- INFO 2026-04-07 02:10:02,504 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
504
- INFO 2026-04-07 02:13:05,975 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
505
- INFO 2026-04-07 02:16:10,140 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
506
- INFO 2026-04-07 02:19:14,240 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
507
- INFO 2026-04-07 02:22:17,137 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
508
- INFO 2026-04-07 02:25:21,416 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
509
- INFO 2026-04-07 02:28:24,738 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
510
- INFO 2026-04-07 02:31:29,706 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
511
- INFO 2026-04-07 02:34:33,435 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
512
- INFO 2026-04-07 02:37:37,740 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
513
- INFO 2026-04-07 02:40:41,387 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
514
- INFO 2026-04-07 02:43:44,863 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
515
- INFO 2026-04-07 02:46:48,385 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
516
- INFO 2026-04-07 02:49:52,438 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
517
- INFO 2026-04-07 02:52:56,831 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
518
- INFO 2026-04-07 02:56:01,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
519
- INFO 2026-04-07 02:59:07,330 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
520
- INFO 2026-04-07 03:02:11,725 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
521
- INFO 2026-04-07 03:05:16,725 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
522
- INFO 2026-04-07 03:08:20,098 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
523
- INFO 2026-04-07 03:11:23,979 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
524
- INFO 2026-04-07 03:14:28,686 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
525
- INFO 2026-04-07 03:17:51,601 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
526
- INFO 2026-04-07 03:18:34,221 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=1
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7792d124f680eb3b1187797c9c324f23e4cf0f643d2787788905cd7f2b83bf2d
3
+ size 63999
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_2.log CHANGED
@@ -1,526 +1,3 @@
1
- INFO 2026-04-05 07:47:39,888 barrel.train.components.optimizers.optimizer_module init_optimizer rank=2, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
2
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
3
- hyperparams=ConfigDict[typing.Any](
4
- lr=5e-05,
5
- weight_decay=0.0,
6
- ),
7
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
8
- torch_optimizer='torch.optim.AdamW',
9
- ), scheduler=SequentialSchedulerConfig(
10
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
11
- milestones=ConfigList[int](
12
- 500,
13
- ),
14
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
15
- warmup=TorchSchedulerConfig(
16
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
17
- hyperparams=ConfigDict[typing.Any](
18
- start_factor=0.0002,
19
- end_factor=1.0,
20
- total_iters=500,
21
- ),
22
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
23
- ),
24
- const=TorchSchedulerConfig(
25
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
26
- hyperparams=ConfigDict[typing.Any](
27
- factor=1.0,
28
- total_iters=300000,
29
- ),
30
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
31
- ),
32
- ),
33
- )
34
- INFO 2026-04-05 07:47:39,890 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
35
- INFO 2026-04-05 07:47:42,059 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge
36
- INFO 2026-04-05 07:47:55,884 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
37
- INFO 2026-04-05 07:48:00,080 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
38
- INFO 2026-04-05 07:48:00,080 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=18021.2, dataset_name=bridge
39
- INFO 2026-04-05 07:48:00,119 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183271, rank=2
40
- INFO 2026-04-05 07:48:01,165 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge rank=2, status=start
41
- INFO 2026-04-05 07:48:01,203 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
42
- INFO 2026-04-05 07:48:01,205 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=21314.9
43
- INFO 2026-04-05 07:48:34,458 barrel.train.core.trainer.trainer session_init status=success, rank=2
44
- INFO 2026-04-05 07:48:34,460 barrel.train.core.trainer.trainer start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
45
- INFO 2026-04-05 07:48:34,538 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-2, worker_pid=3875756, parent_pid=3872891, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
46
- INFO 2026-04-05 07:48:34,582 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-2, worker_pid=3875804, parent_pid=3872891, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
47
- INFO 2026-04-05 07:51:52,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
48
- INFO 2026-04-05 07:54:44,997 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
49
- INFO 2026-04-05 07:57:35,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
50
- INFO 2026-04-05 08:00:26,862 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
51
- INFO 2026-04-05 08:03:18,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
52
- INFO 2026-04-05 08:06:09,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
53
- INFO 2026-04-05 08:09:01,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
54
- INFO 2026-04-05 08:11:53,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
55
- INFO 2026-04-05 08:14:44,578 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
56
- INFO 2026-04-05 08:17:37,375 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
57
- INFO 2026-04-05 08:20:30,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
58
- INFO 2026-04-05 08:23:23,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
59
- INFO 2026-04-05 08:26:15,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
60
- INFO 2026-04-05 08:29:07,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
61
- INFO 2026-04-05 08:31:59,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
62
- INFO 2026-04-05 08:34:50,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
63
- INFO 2026-04-05 08:37:43,456 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
64
- INFO 2026-04-05 08:40:35,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
65
- INFO 2026-04-05 08:43:28,467 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
66
- INFO 2026-04-05 08:46:21,132 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
67
- INFO 2026-04-05 08:49:13,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
68
- INFO 2026-04-05 08:52:04,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
69
- INFO 2026-04-05 08:54:56,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
70
- INFO 2026-04-05 08:57:48,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
71
- INFO 2026-04-05 09:00:40,238 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
72
- INFO 2026-04-05 09:03:32,325 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
73
- INFO 2026-04-05 09:06:23,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
74
- INFO 2026-04-05 09:09:15,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
75
- INFO 2026-04-05 09:12:08,117 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
76
- INFO 2026-04-05 09:14:59,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
77
- INFO 2026-04-05 09:17:51,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
78
- INFO 2026-04-05 09:20:42,298 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
79
- INFO 2026-04-05 09:23:34,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
80
- INFO 2026-04-05 09:26:26,063 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
81
- INFO 2026-04-05 09:29:18,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
82
- INFO 2026-04-05 09:32:09,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
83
- INFO 2026-04-05 09:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
84
- INFO 2026-04-05 09:37:52,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
85
- INFO 2026-04-05 09:40:43,428 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
86
- INFO 2026-04-05 09:43:34,586 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
87
- INFO 2026-04-05 09:46:26,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
88
- INFO 2026-04-05 09:49:17,267 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
89
- INFO 2026-04-05 09:52:06,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
90
- INFO 2026-04-05 09:54:57,889 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
91
- INFO 2026-04-05 09:57:48,894 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
92
- INFO 2026-04-05 10:00:41,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
93
- INFO 2026-04-05 10:03:33,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
94
- INFO 2026-04-05 10:06:24,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
95
- INFO 2026-04-05 10:09:17,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
96
- INFO 2026-04-05 10:12:18,686 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
97
- INFO 2026-04-05 10:15:33,482 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
98
- INFO 2026-04-05 10:18:24,744 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
99
- INFO 2026-04-05 10:21:17,302 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
100
- INFO 2026-04-05 10:24:08,917 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
101
- INFO 2026-04-05 10:27:00,005 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
102
- INFO 2026-04-05 10:29:51,958 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
103
- INFO 2026-04-05 10:32:42,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
104
- INFO 2026-04-05 10:35:34,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
105
- INFO 2026-04-05 10:38:25,583 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
106
- INFO 2026-04-05 10:41:18,350 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
107
- INFO 2026-04-05 10:44:08,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
108
- INFO 2026-04-05 10:47:00,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
109
- INFO 2026-04-05 10:49:51,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
110
- INFO 2026-04-05 10:52:43,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
111
- INFO 2026-04-05 10:55:35,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
112
- INFO 2026-04-05 10:58:26,949 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
113
- INFO 2026-04-05 11:01:18,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
114
- INFO 2026-04-05 11:04:10,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
115
- INFO 2026-04-05 11:07:02,461 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
116
- INFO 2026-04-05 11:09:53,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
117
- INFO 2026-04-05 11:12:45,451 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
118
- INFO 2026-04-05 11:15:36,492 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
119
- INFO 2026-04-05 11:18:28,400 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
120
- INFO 2026-04-05 11:21:20,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
121
- INFO 2026-04-05 11:24:11,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
122
- INFO 2026-04-05 11:27:02,198 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
123
- INFO 2026-04-05 11:29:53,736 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
124
- INFO 2026-04-05 11:32:44,011 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
125
- INFO 2026-04-05 11:35:35,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
126
- INFO 2026-04-05 11:38:25,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
127
- INFO 2026-04-05 11:41:15,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
128
- INFO 2026-04-05 11:44:05,134 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
129
- INFO 2026-04-05 11:46:56,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
130
- INFO 2026-04-05 11:49:47,469 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
131
- INFO 2026-04-05 11:52:38,563 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
132
- INFO 2026-04-05 11:55:30,290 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
133
- INFO 2026-04-05 11:58:21,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
134
- INFO 2026-04-05 12:01:13,424 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
135
- INFO 2026-04-05 12:04:04,015 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
136
- INFO 2026-04-05 12:06:55,195 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
137
- INFO 2026-04-05 12:09:46,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
138
- INFO 2026-04-05 12:12:37,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
139
- INFO 2026-04-05 12:15:28,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
140
- INFO 2026-04-05 12:18:19,903 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
141
- INFO 2026-04-05 12:21:11,565 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
142
- INFO 2026-04-05 12:24:02,399 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
143
- INFO 2026-04-05 12:26:54,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
144
- INFO 2026-04-05 12:29:45,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
145
- INFO 2026-04-05 12:32:36,286 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
146
- INFO 2026-04-05 12:35:36,776 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
147
- INFO 2026-04-05 12:38:50,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
148
- INFO 2026-04-05 12:41:41,343 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
149
- INFO 2026-04-05 12:44:32,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
150
- INFO 2026-04-05 12:47:23,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
151
- INFO 2026-04-05 12:50:13,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
152
- INFO 2026-04-05 12:53:06,274 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
153
- INFO 2026-04-05 12:55:56,979 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
154
- INFO 2026-04-05 12:58:48,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
155
- INFO 2026-04-05 13:01:39,887 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
156
- INFO 2026-04-05 13:04:31,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
157
- INFO 2026-04-05 13:07:22,319 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
158
- INFO 2026-04-05 13:10:15,398 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
159
- INFO 2026-04-05 13:13:06,519 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
160
- INFO 2026-04-05 13:15:57,164 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
161
- INFO 2026-04-05 13:18:49,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
162
- INFO 2026-04-05 13:21:40,162 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
163
- INFO 2026-04-05 13:24:31,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
164
- INFO 2026-04-05 13:27:22,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
165
- INFO 2026-04-05 13:30:13,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
166
- INFO 2026-04-05 13:33:06,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
167
- INFO 2026-04-05 13:35:57,606 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
168
- INFO 2026-04-05 13:38:47,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
169
- INFO 2026-04-05 13:41:40,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
170
- INFO 2026-04-05 13:44:31,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
171
- INFO 2026-04-05 13:47:23,254 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
172
- INFO 2026-04-05 13:50:14,660 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
173
- INFO 2026-04-05 13:53:06,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
174
- INFO 2026-04-05 13:55:58,107 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
175
- INFO 2026-04-05 13:58:50,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
176
- INFO 2026-04-05 14:01:41,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
177
- INFO 2026-04-05 14:04:32,278 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
178
- INFO 2026-04-05 14:07:24,300 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
179
- INFO 2026-04-05 14:10:15,785 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
180
- INFO 2026-04-05 14:13:07,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
181
- INFO 2026-04-05 14:15:58,762 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
182
- INFO 2026-04-05 14:18:49,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
183
- INFO 2026-04-05 14:21:40,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
184
- INFO 2026-04-05 14:24:31,803 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
185
- INFO 2026-04-05 14:27:22,607 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
186
- INFO 2026-04-05 14:30:14,379 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
187
- INFO 2026-04-05 14:33:06,049 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
188
- INFO 2026-04-05 14:35:56,094 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
189
- INFO 2026-04-05 14:38:48,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
190
- INFO 2026-04-05 14:41:40,352 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
191
- INFO 2026-04-05 14:44:32,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
192
- INFO 2026-04-05 14:47:25,214 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
193
- INFO 2026-04-05 14:50:15,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
194
- INFO 2026-04-05 14:53:07,347 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
195
- INFO 2026-04-05 14:56:00,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
196
- INFO 2026-04-05 14:59:02,183 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
197
- INFO 2026-04-05 14:59:35,376 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=2
198
- INFO 2026-04-06 07:33:53,119 barrel.train.components.optimizers.optimizer_module init_optimizer rank=2, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
199
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
200
- hyperparams=ConfigDict[typing.Any](
201
- lr=5e-05,
202
- weight_decay=0.0,
203
- ),
204
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
205
- torch_optimizer='torch.optim.AdamW',
206
- ), scheduler=SequentialSchedulerConfig(
207
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
208
- milestones=ConfigList[int](
209
- 500,
210
- ),
211
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
212
- warmup=TorchSchedulerConfig(
213
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
214
- hyperparams=ConfigDict[typing.Any](
215
- start_factor=0.0002,
216
- end_factor=1.0,
217
- total_iters=500,
218
- ),
219
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
220
- ),
221
- const=TorchSchedulerConfig(
222
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
223
- hyperparams=ConfigDict[typing.Any](
224
- factor=1.0,
225
- total_iters=300000,
226
- ),
227
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
228
- ),
229
- ),
230
- )
231
- INFO 2026-04-06 07:33:53,122 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
232
- INFO 2026-04-06 07:33:56,084 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
233
- INFO 2026-04-06 07:34:13,383 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
234
- INFO 2026-04-06 07:34:20,583 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
235
- INFO 2026-04-06 07:34:25,333 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
236
- INFO 2026-04-06 07:34:25,333 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=29248.8, dataset_name=bridge_paraphrase_k5
237
- INFO 2026-04-06 07:34:25,374 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183271, rank=2
238
- INFO 2026-04-06 07:34:26,288 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=2, status=start
239
- INFO 2026-04-06 07:34:26,333 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
240
- INFO 2026-04-06 07:34:26,334 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=33212.6
241
- INFO 2026-04-06 07:34:26,340 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=2, status=start
242
- INFO 2026-04-06 07:36:05,842 barrel.train.core.trainer.trainer Restore session checkpoint rank=2, status=success
243
- INFO 2026-04-06 07:36:27,876 barrel.train.core.trainer.trainer session_init status=success, rank=2
244
- INFO 2026-04-06 07:36:27,879 barrel.train.core.trainer.trainer start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
245
- INFO 2026-04-06 07:36:27,947 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=2465174, parent_pid=2461887, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
246
- INFO 2026-04-06 07:36:28,002 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=2465214, parent_pid=2461887, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
247
- INFO 2026-04-06 07:39:43,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
248
- INFO 2026-04-06 07:42:35,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
249
- INFO 2026-04-06 07:45:26,710 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
250
- INFO 2026-04-06 07:48:18,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
251
- INFO 2026-04-06 07:51:08,588 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
252
- INFO 2026-04-06 07:54:00,463 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
253
- INFO 2026-04-06 07:56:51,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
254
- INFO 2026-04-06 07:59:43,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
255
- INFO 2026-04-06 08:02:34,655 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
256
- INFO 2026-04-06 08:05:26,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
257
- INFO 2026-04-06 08:08:18,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
258
- INFO 2026-04-06 08:11:11,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
259
- INFO 2026-04-06 08:14:03,149 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
260
- INFO 2026-04-06 08:16:57,119 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
261
- INFO 2026-04-06 08:19:48,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
262
- INFO 2026-04-06 08:22:39,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
263
- INFO 2026-04-06 08:25:31,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
264
- INFO 2026-04-06 08:28:25,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
265
- INFO 2026-04-06 08:31:16,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
266
- INFO 2026-04-06 08:34:09,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
267
- INFO 2026-04-06 08:37:01,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
268
- INFO 2026-04-06 08:39:53,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
269
- INFO 2026-04-06 08:42:45,381 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
270
- INFO 2026-04-06 08:45:38,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
271
- INFO 2026-04-06 08:48:30,988 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
272
- INFO 2026-04-06 08:51:23,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
273
- INFO 2026-04-06 08:54:13,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
274
- INFO 2026-04-06 08:57:05,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
275
- INFO 2026-04-06 08:59:57,948 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
276
- INFO 2026-04-06 09:02:49,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
277
- INFO 2026-04-06 09:05:41,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
278
- INFO 2026-04-06 09:08:33,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
279
- INFO 2026-04-06 09:11:25,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
280
- INFO 2026-04-06 09:14:17,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
281
- INFO 2026-04-06 09:17:08,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
282
- INFO 2026-04-06 09:20:01,792 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
283
- INFO 2026-04-06 09:22:53,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
284
- INFO 2026-04-06 09:25:46,358 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
285
- INFO 2026-04-06 09:28:40,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
286
- INFO 2026-04-06 09:31:33,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
287
- INFO 2026-04-06 09:34:24,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
288
- INFO 2026-04-06 09:37:17,501 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
289
- INFO 2026-04-06 09:40:08,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
290
- INFO 2026-04-06 09:43:00,698 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
291
- INFO 2026-04-06 09:45:52,659 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
292
- INFO 2026-04-06 09:48:44,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
293
- INFO 2026-04-06 09:51:36,495 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
294
- INFO 2026-04-06 09:54:29,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
295
- INFO 2026-04-06 09:57:20,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
296
- INFO 2026-04-06 10:00:22,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
297
- INFO 2026-04-06 10:03:48,881 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
298
- INFO 2026-04-06 10:06:52,081 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
299
- INFO 2026-04-06 10:09:59,023 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
300
- INFO 2026-04-06 10:13:05,837 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
301
- INFO 2026-04-06 10:16:11,929 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
302
- INFO 2026-04-06 10:19:15,977 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
303
- INFO 2026-04-06 10:22:24,664 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
304
- INFO 2026-04-06 10:25:33,549 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
305
- INFO 2026-04-06 10:28:42,172 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
306
- INFO 2026-04-06 10:31:51,516 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
307
- INFO 2026-04-06 10:35:00,444 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
308
- INFO 2026-04-06 10:38:08,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
309
- INFO 2026-04-06 10:41:15,601 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
310
- INFO 2026-04-06 11:29:31,533 barrel.train.components.optimizers.optimizer_module init_optimizer rank=2, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
311
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
312
- hyperparams=ConfigDict[typing.Any](
313
- lr=5e-05,
314
- weight_decay=0.0,
315
- ),
316
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
317
- torch_optimizer='torch.optim.AdamW',
318
- ), scheduler=SequentialSchedulerConfig(
319
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
320
- milestones=ConfigList[int](
321
- 500,
322
- ),
323
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
324
- warmup=TorchSchedulerConfig(
325
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
326
- hyperparams=ConfigDict[typing.Any](
327
- start_factor=0.0002,
328
- end_factor=1.0,
329
- total_iters=500,
330
- ),
331
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
332
- ),
333
- const=TorchSchedulerConfig(
334
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
335
- hyperparams=ConfigDict[typing.Any](
336
- factor=1.0,
337
- total_iters=300000,
338
- ),
339
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
340
- ),
341
- ),
342
- )
343
- INFO 2026-04-06 11:29:31,537 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
344
- INFO 2026-04-06 11:29:34,940 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
345
- INFO 2026-04-06 11:29:43,630 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
346
- INFO 2026-04-06 11:29:47,887 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
347
- INFO 2026-04-06 11:29:51,559 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
348
- INFO 2026-04-06 11:29:51,559 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=16619.3, dataset_name=bridge_paraphrase_k5
349
- INFO 2026-04-06 11:29:51,600 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183271, rank=2
350
- INFO 2026-04-06 11:29:52,408 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=2, status=start
351
- INFO 2026-04-06 11:29:52,447 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
352
- INFO 2026-04-06 11:29:52,448 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=20911.5
353
- INFO 2026-04-06 11:29:52,453 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=2, status=start
354
- INFO 2026-04-06 11:31:39,838 barrel.train.core.trainer.trainer Restore session checkpoint rank=2, status=success
355
- INFO 2026-04-06 11:32:01,842 barrel.train.core.trainer.trainer session_init status=success, rank=2
356
- INFO 2026-04-06 11:32:01,844 barrel.train.core.trainer.trainer start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
357
- INFO 2026-04-06 11:32:02,006 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-7, worker_pid=3492777, parent_pid=3489442, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
358
- INFO 2026-04-06 11:32:02,026 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-7, worker_pid=3492786, parent_pid=3489442, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
359
- INFO 2026-04-06 11:35:29,328 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
360
- INFO 2026-04-06 11:38:35,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
361
- INFO 2026-04-06 11:41:41,427 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
362
- INFO 2026-04-06 11:44:46,852 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
363
- INFO 2026-04-06 11:47:52,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
364
- INFO 2026-04-06 11:50:56,952 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
365
- INFO 2026-04-06 11:54:03,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
366
- INFO 2026-04-06 11:57:07,420 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
367
- INFO 2026-04-06 12:00:13,670 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
368
- INFO 2026-04-06 12:03:19,351 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
369
- INFO 2026-04-06 12:06:23,695 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
370
- INFO 2026-04-06 12:09:29,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
371
- INFO 2026-04-06 12:12:35,569 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
372
- INFO 2026-04-06 12:15:41,416 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
373
- INFO 2026-04-06 12:18:46,702 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
374
- INFO 2026-04-06 12:21:51,861 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
375
- INFO 2026-04-06 12:24:57,746 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
376
- INFO 2026-04-06 12:28:01,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
377
- INFO 2026-04-06 12:31:07,224 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
378
- INFO 2026-04-06 12:34:11,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
379
- INFO 2026-04-06 12:37:16,738 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
380
- INFO 2026-04-06 12:40:21,215 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
381
- INFO 2026-04-06 12:43:25,432 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
382
- INFO 2026-04-06 12:46:31,790 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
383
- INFO 2026-04-06 12:49:37,505 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
384
- INFO 2026-04-06 12:52:43,061 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
385
- INFO 2026-04-06 12:55:47,771 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
386
- INFO 2026-04-06 12:58:53,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
387
- INFO 2026-04-06 13:01:58,051 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
388
- INFO 2026-04-06 13:05:03,711 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
389
- INFO 2026-04-06 13:08:09,155 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
390
- INFO 2026-04-06 13:11:13,195 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
391
- INFO 2026-04-06 13:14:19,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
392
- INFO 2026-04-06 13:17:25,217 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
393
- INFO 2026-04-06 13:20:30,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
394
- INFO 2026-04-06 13:23:35,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
395
- INFO 2026-04-06 13:26:41,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
396
- INFO 2026-04-06 13:29:48,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
397
- INFO 2026-04-06 13:32:52,616 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
398
- INFO 2026-04-06 13:35:57,628 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
399
- INFO 2026-04-06 13:39:03,078 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
400
- INFO 2026-04-06 13:42:10,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
401
- INFO 2026-04-06 13:45:14,630 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
402
- INFO 2026-04-06 13:48:20,967 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
403
- INFO 2026-04-06 13:51:25,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
404
- INFO 2026-04-06 13:54:30,720 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
405
- INFO 2026-04-06 13:57:36,001 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
406
- INFO 2026-04-06 14:00:41,873 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
407
- INFO 2026-04-06 14:03:49,030 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
408
- INFO 2026-04-06 14:07:04,710 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
409
- INFO 2026-04-06 14:10:39,270 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
410
- INFO 2026-04-06 14:13:44,082 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
411
- INFO 2026-04-06 14:16:49,280 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
412
- INFO 2026-04-06 14:19:53,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
413
- INFO 2026-04-06 14:22:58,775 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
414
- INFO 2026-04-06 14:26:03,758 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
415
- INFO 2026-04-06 14:29:07,810 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
416
- INFO 2026-04-06 14:32:11,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
417
- INFO 2026-04-06 14:35:16,304 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
418
- INFO 2026-04-06 14:38:22,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
419
- INFO 2026-04-06 14:41:25,471 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
420
- INFO 2026-04-06 14:44:30,524 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
421
- INFO 2026-04-06 14:47:36,041 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
422
- INFO 2026-04-06 14:50:40,764 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
423
- INFO 2026-04-06 14:53:45,031 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
424
- INFO 2026-04-06 14:56:50,250 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
425
- INFO 2026-04-06 14:59:56,683 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
426
- INFO 2026-04-06 15:03:02,331 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
427
- INFO 2026-04-07 00:41:05,445 barrel.train.components.optimizers.optimizer_module init_optimizer rank=2, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
428
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
429
- hyperparams=ConfigDict[typing.Any](
430
- lr=5e-05,
431
- weight_decay=0.0,
432
- ),
433
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
434
- torch_optimizer='torch.optim.AdamW',
435
- ), scheduler=SequentialSchedulerConfig(
436
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
437
- milestones=ConfigList[int](
438
- 500,
439
- ),
440
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
441
- warmup=TorchSchedulerConfig(
442
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
443
- hyperparams=ConfigDict[typing.Any](
444
- start_factor=0.0002,
445
- end_factor=1.0,
446
- total_iters=500,
447
- ),
448
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
449
- ),
450
- const=TorchSchedulerConfig(
451
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
452
- hyperparams=ConfigDict[typing.Any](
453
- factor=1.0,
454
- total_iters=300000,
455
- ),
456
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
457
- ),
458
- ),
459
- )
460
- INFO 2026-04-07 00:41:05,450 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
461
- INFO 2026-04-07 00:41:08,566 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
462
- INFO 2026-04-07 00:41:17,724 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
463
- INFO 2026-04-07 00:41:22,571 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
464
- INFO 2026-04-07 00:41:26,705 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
465
- INFO 2026-04-07 00:41:26,705 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=18138.6, dataset_name=bridge_paraphrase_k5
466
- INFO 2026-04-07 00:41:26,745 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183271, rank=2
467
- INFO 2026-04-07 00:41:27,734 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=2, status=start
468
- INFO 2026-04-07 00:41:27,773 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
469
- INFO 2026-04-07 00:41:27,774 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=22324.7
470
- INFO 2026-04-07 00:41:27,779 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=2, status=start
471
- INFO 2026-04-07 00:43:48,420 barrel.train.core.trainer.trainer Restore session checkpoint rank=2, status=success
472
- INFO 2026-04-07 00:44:10,421 barrel.train.core.trainer.trainer session_init status=success, rank=2
473
- INFO 2026-04-07 00:44:10,423 barrel.train.core.trainer.trainer start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
474
- INFO 2026-04-07 00:44:10,474 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=3313509, parent_pid=3310180, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
475
- INFO 2026-04-07 00:44:10,497 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=3313514, parent_pid=3310180, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
476
- INFO 2026-04-07 00:47:36,025 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
477
- INFO 2026-04-07 00:50:38,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
478
- INFO 2026-04-07 00:53:42,312 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
479
- INFO 2026-04-07 00:56:45,082 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
480
- INFO 2026-04-07 00:59:47,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
481
- INFO 2026-04-07 01:02:50,207 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
482
- INFO 2026-04-07 01:05:53,527 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
483
- INFO 2026-04-07 01:08:56,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
484
- INFO 2026-04-07 01:11:58,574 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
485
- INFO 2026-04-07 01:15:02,202 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
486
- INFO 2026-04-07 01:18:05,103 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
487
- INFO 2026-04-07 01:21:08,472 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
488
- INFO 2026-04-07 01:24:11,705 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
489
- INFO 2026-04-07 01:27:15,142 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
490
- INFO 2026-04-07 01:30:16,884 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
491
- INFO 2026-04-07 01:33:19,433 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
492
- INFO 2026-04-07 01:36:22,454 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
493
- INFO 2026-04-07 01:39:25,140 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
494
- INFO 2026-04-07 01:42:29,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
495
- INFO 2026-04-07 01:45:32,636 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
496
- INFO 2026-04-07 01:48:36,837 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
497
- INFO 2026-04-07 01:51:39,489 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
498
- INFO 2026-04-07 01:54:42,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
499
- INFO 2026-04-07 01:57:46,408 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
500
- INFO 2026-04-07 02:00:50,168 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
501
- INFO 2026-04-07 02:03:54,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
502
- INFO 2026-04-07 02:06:57,974 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
503
- INFO 2026-04-07 02:10:02,506 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
504
- INFO 2026-04-07 02:13:05,974 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
505
- INFO 2026-04-07 02:16:10,139 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
506
- INFO 2026-04-07 02:19:14,241 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
507
- INFO 2026-04-07 02:22:17,137 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
508
- INFO 2026-04-07 02:25:21,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
509
- INFO 2026-04-07 02:28:24,735 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
510
- INFO 2026-04-07 02:31:29,705 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
511
- INFO 2026-04-07 02:34:33,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
512
- INFO 2026-04-07 02:37:37,738 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
513
- INFO 2026-04-07 02:40:41,387 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
514
- INFO 2026-04-07 02:43:44,849 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
515
- INFO 2026-04-07 02:46:48,384 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
516
- INFO 2026-04-07 02:49:52,436 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
517
- INFO 2026-04-07 02:52:56,831 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
518
- INFO 2026-04-07 02:56:01,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
519
- INFO 2026-04-07 02:59:07,327 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
520
- INFO 2026-04-07 03:02:11,724 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
521
- INFO 2026-04-07 03:05:16,726 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
522
- INFO 2026-04-07 03:08:20,100 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
523
- INFO 2026-04-07 03:11:23,978 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
524
- INFO 2026-04-07 03:14:28,686 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
525
- INFO 2026-04-07 03:17:51,379 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
526
- INFO 2026-04-07 03:18:34,222 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=2
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aaa6984aa5db79c0d8b484f905b51cc55b5213c8c95420d7d614c9d54f45fd90
3
+ size 63999
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_3.log CHANGED
@@ -1,526 +1,3 @@
1
- INFO 2026-04-05 07:47:39,920 barrel.train.components.optimizers.optimizer_module init_optimizer rank=3, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
2
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
3
- hyperparams=ConfigDict[typing.Any](
4
- lr=5e-05,
5
- weight_decay=0.0,
6
- ),
7
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
8
- torch_optimizer='torch.optim.AdamW',
9
- ), scheduler=SequentialSchedulerConfig(
10
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
11
- milestones=ConfigList[int](
12
- 500,
13
- ),
14
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
15
- warmup=TorchSchedulerConfig(
16
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
17
- hyperparams=ConfigDict[typing.Any](
18
- start_factor=0.0002,
19
- end_factor=1.0,
20
- total_iters=500,
21
- ),
22
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
23
- ),
24
- const=TorchSchedulerConfig(
25
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
26
- hyperparams=ConfigDict[typing.Any](
27
- factor=1.0,
28
- total_iters=300000,
29
- ),
30
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
31
- ),
32
- ),
33
- )
34
- INFO 2026-04-05 07:47:39,921 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
35
- INFO 2026-04-05 07:47:42,035 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge
36
- INFO 2026-04-05 07:48:03,688 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
37
- INFO 2026-04-05 07:48:08,057 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
38
- INFO 2026-04-05 07:48:08,057 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=26022.5, dataset_name=bridge
39
- INFO 2026-04-05 07:48:08,097 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183231, rank=3
40
- INFO 2026-04-05 07:48:08,968 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge rank=3, status=start
41
- INFO 2026-04-05 07:48:09,007 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
42
- INFO 2026-04-05 07:48:09,009 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=29087.800000000003
43
- INFO 2026-04-05 07:48:34,458 barrel.train.core.trainer.trainer session_init status=success, rank=3
44
- INFO 2026-04-05 07:48:34,460 barrel.train.core.trainer.trainer start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
45
- INFO 2026-04-05 07:48:34,516 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-2, worker_pid=3875749, parent_pid=3872892, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
46
- INFO 2026-04-05 07:48:34,547 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-2, worker_pid=3875761, parent_pid=3872892, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
47
- INFO 2026-04-05 07:51:52,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
48
- INFO 2026-04-05 07:54:44,997 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
49
- INFO 2026-04-05 07:57:35,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
50
- INFO 2026-04-05 08:00:26,862 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
51
- INFO 2026-04-05 08:03:18,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
52
- INFO 2026-04-05 08:06:09,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
53
- INFO 2026-04-05 08:09:01,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
54
- INFO 2026-04-05 08:11:53,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
55
- INFO 2026-04-05 08:14:44,578 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
56
- INFO 2026-04-05 08:17:37,375 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
57
- INFO 2026-04-05 08:20:30,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
58
- INFO 2026-04-05 08:23:23,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
59
- INFO 2026-04-05 08:26:15,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
60
- INFO 2026-04-05 08:29:07,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
61
- INFO 2026-04-05 08:31:59,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
62
- INFO 2026-04-05 08:34:50,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
63
- INFO 2026-04-05 08:37:43,456 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
64
- INFO 2026-04-05 08:40:35,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
65
- INFO 2026-04-05 08:43:28,467 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
66
- INFO 2026-04-05 08:46:21,132 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
67
- INFO 2026-04-05 08:49:13,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
68
- INFO 2026-04-05 08:52:04,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
69
- INFO 2026-04-05 08:54:56,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
70
- INFO 2026-04-05 08:57:48,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
71
- INFO 2026-04-05 09:00:40,238 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
72
- INFO 2026-04-05 09:03:32,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
73
- INFO 2026-04-05 09:06:23,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
74
- INFO 2026-04-05 09:09:15,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
75
- INFO 2026-04-05 09:12:08,117 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
76
- INFO 2026-04-05 09:14:59,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
77
- INFO 2026-04-05 09:17:51,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
78
- INFO 2026-04-05 09:20:42,298 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
79
- INFO 2026-04-05 09:23:34,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
80
- INFO 2026-04-05 09:26:26,063 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
81
- INFO 2026-04-05 09:29:18,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
82
- INFO 2026-04-05 09:32:09,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
83
- INFO 2026-04-05 09:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
84
- INFO 2026-04-05 09:37:52,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
85
- INFO 2026-04-05 09:40:43,428 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
86
- INFO 2026-04-05 09:43:34,586 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
87
- INFO 2026-04-05 09:46:26,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
88
- INFO 2026-04-05 09:49:17,267 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
89
- INFO 2026-04-05 09:52:06,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
90
- INFO 2026-04-05 09:54:57,889 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
91
- INFO 2026-04-05 09:57:48,894 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
92
- INFO 2026-04-05 10:00:41,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
93
- INFO 2026-04-05 10:03:33,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
94
- INFO 2026-04-05 10:06:24,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
95
- INFO 2026-04-05 10:09:17,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
96
- INFO 2026-04-05 10:12:18,806 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
97
- INFO 2026-04-05 10:15:33,483 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
98
- INFO 2026-04-05 10:18:24,744 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
99
- INFO 2026-04-05 10:21:17,302 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
100
- INFO 2026-04-05 10:24:08,917 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
101
- INFO 2026-04-05 10:27:00,005 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
102
- INFO 2026-04-05 10:29:51,959 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
103
- INFO 2026-04-05 10:32:42,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
104
- INFO 2026-04-05 10:35:34,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
105
- INFO 2026-04-05 10:38:25,583 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
106
- INFO 2026-04-05 10:41:18,350 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
107
- INFO 2026-04-05 10:44:08,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
108
- INFO 2026-04-05 10:47:00,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
109
- INFO 2026-04-05 10:49:51,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
110
- INFO 2026-04-05 10:52:43,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
111
- INFO 2026-04-05 10:55:35,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
112
- INFO 2026-04-05 10:58:26,949 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
113
- INFO 2026-04-05 11:01:18,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
114
- INFO 2026-04-05 11:04:10,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
115
- INFO 2026-04-05 11:07:02,461 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
116
- INFO 2026-04-05 11:09:53,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
117
- INFO 2026-04-05 11:12:45,451 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
118
- INFO 2026-04-05 11:15:36,492 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
119
- INFO 2026-04-05 11:18:28,400 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
120
- INFO 2026-04-05 11:21:20,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
121
- INFO 2026-04-05 11:24:11,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
122
- INFO 2026-04-05 11:27:02,198 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
123
- INFO 2026-04-05 11:29:53,736 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
124
- INFO 2026-04-05 11:32:44,011 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
125
- INFO 2026-04-05 11:35:35,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
126
- INFO 2026-04-05 11:38:25,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
127
- INFO 2026-04-05 11:41:15,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
128
- INFO 2026-04-05 11:44:05,134 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
129
- INFO 2026-04-05 11:46:56,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
130
- INFO 2026-04-05 11:49:47,469 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
131
- INFO 2026-04-05 11:52:38,563 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
132
- INFO 2026-04-05 11:55:30,289 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
133
- INFO 2026-04-05 11:58:21,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
134
- INFO 2026-04-05 12:01:13,424 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
135
- INFO 2026-04-05 12:04:04,015 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
136
- INFO 2026-04-05 12:06:55,196 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
137
- INFO 2026-04-05 12:09:46,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
138
- INFO 2026-04-05 12:12:37,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
139
- INFO 2026-04-05 12:15:28,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
140
- INFO 2026-04-05 12:18:19,903 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
141
- INFO 2026-04-05 12:21:11,565 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
142
- INFO 2026-04-05 12:24:02,399 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
143
- INFO 2026-04-05 12:26:54,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
144
- INFO 2026-04-05 12:29:45,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
145
- INFO 2026-04-05 12:32:36,286 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
146
- INFO 2026-04-05 12:35:36,955 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
147
- INFO 2026-04-05 12:38:50,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
148
- INFO 2026-04-05 12:41:41,343 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
149
- INFO 2026-04-05 12:44:32,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
150
- INFO 2026-04-05 12:47:23,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
151
- INFO 2026-04-05 12:50:13,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
152
- INFO 2026-04-05 12:53:06,274 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
153
- INFO 2026-04-05 12:55:56,979 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
154
- INFO 2026-04-05 12:58:48,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
155
- INFO 2026-04-05 13:01:39,887 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
156
- INFO 2026-04-05 13:04:31,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
157
- INFO 2026-04-05 13:07:22,319 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
158
- INFO 2026-04-05 13:10:15,398 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
159
- INFO 2026-04-05 13:13:06,519 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
160
- INFO 2026-04-05 13:15:57,164 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
161
- INFO 2026-04-05 13:18:49,647 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
162
- INFO 2026-04-05 13:21:40,162 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
163
- INFO 2026-04-05 13:24:31,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
164
- INFO 2026-04-05 13:27:22,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
165
- INFO 2026-04-05 13:30:13,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
166
- INFO 2026-04-05 13:33:06,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
167
- INFO 2026-04-05 13:35:57,606 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
168
- INFO 2026-04-05 13:38:47,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
169
- INFO 2026-04-05 13:41:40,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
170
- INFO 2026-04-05 13:44:31,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
171
- INFO 2026-04-05 13:47:23,254 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
172
- INFO 2026-04-05 13:50:14,660 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
173
- INFO 2026-04-05 13:53:06,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
174
- INFO 2026-04-05 13:55:58,107 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
175
- INFO 2026-04-05 13:58:50,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
176
- INFO 2026-04-05 14:01:41,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
177
- INFO 2026-04-05 14:04:32,278 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
178
- INFO 2026-04-05 14:07:24,300 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
179
- INFO 2026-04-05 14:10:15,785 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
180
- INFO 2026-04-05 14:13:07,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
181
- INFO 2026-04-05 14:15:58,762 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
182
- INFO 2026-04-05 14:18:49,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
183
- INFO 2026-04-05 14:21:40,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
184
- INFO 2026-04-05 14:24:31,803 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
185
- INFO 2026-04-05 14:27:22,607 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
186
- INFO 2026-04-05 14:30:14,379 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
187
- INFO 2026-04-05 14:33:06,049 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
188
- INFO 2026-04-05 14:35:56,094 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
189
- INFO 2026-04-05 14:38:48,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
190
- INFO 2026-04-05 14:41:40,352 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
191
- INFO 2026-04-05 14:44:32,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
192
- INFO 2026-04-05 14:47:25,215 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
193
- INFO 2026-04-05 14:50:15,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
194
- INFO 2026-04-05 14:53:07,347 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
195
- INFO 2026-04-05 14:56:00,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
196
- INFO 2026-04-05 14:59:01,670 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
197
- INFO 2026-04-05 14:59:35,376 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=3
198
- INFO 2026-04-06 07:33:53,166 barrel.train.components.optimizers.optimizer_module init_optimizer rank=3, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
199
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
200
- hyperparams=ConfigDict[typing.Any](
201
- lr=5e-05,
202
- weight_decay=0.0,
203
- ),
204
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
205
- torch_optimizer='torch.optim.AdamW',
206
- ), scheduler=SequentialSchedulerConfig(
207
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
208
- milestones=ConfigList[int](
209
- 500,
210
- ),
211
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
212
- warmup=TorchSchedulerConfig(
213
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
214
- hyperparams=ConfigDict[typing.Any](
215
- start_factor=0.0002,
216
- end_factor=1.0,
217
- total_iters=500,
218
- ),
219
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
220
- ),
221
- const=TorchSchedulerConfig(
222
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
223
- hyperparams=ConfigDict[typing.Any](
224
- factor=1.0,
225
- total_iters=300000,
226
- ),
227
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
228
- ),
229
- ),
230
- )
231
- INFO 2026-04-06 07:33:53,171 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
232
- INFO 2026-04-06 07:33:56,071 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
233
- INFO 2026-04-06 07:34:10,494 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
234
- INFO 2026-04-06 07:34:18,352 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
235
- INFO 2026-04-06 07:34:22,693 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
236
- INFO 2026-04-06 07:34:22,694 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=26622.699999999997, dataset_name=bridge_paraphrase_k5
237
- INFO 2026-04-06 07:34:22,735 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183231, rank=3
238
- INFO 2026-04-06 07:34:23,620 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=3, status=start
239
- INFO 2026-04-06 07:34:23,662 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
240
- INFO 2026-04-06 07:34:23,664 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=30493.0
241
- INFO 2026-04-06 07:34:23,671 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=3, status=start
242
- INFO 2026-04-06 07:36:05,858 barrel.train.core.trainer.trainer Restore session checkpoint rank=3, status=success
243
- INFO 2026-04-06 07:36:27,877 barrel.train.core.trainer.trainer session_init status=success, rank=3
244
- INFO 2026-04-06 07:36:27,879 barrel.train.core.trainer.trainer start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
245
- INFO 2026-04-06 07:36:27,940 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=2465170, parent_pid=2461888, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
246
- INFO 2026-04-06 07:36:27,974 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=2465197, parent_pid=2461888, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
247
- INFO 2026-04-06 07:39:43,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
248
- INFO 2026-04-06 07:42:35,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
249
- INFO 2026-04-06 07:45:26,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
250
- INFO 2026-04-06 07:48:18,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
251
- INFO 2026-04-06 07:51:08,588 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
252
- INFO 2026-04-06 07:54:00,463 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
253
- INFO 2026-04-06 07:56:51,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
254
- INFO 2026-04-06 07:59:43,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
255
- INFO 2026-04-06 08:02:34,656 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
256
- INFO 2026-04-06 08:05:26,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
257
- INFO 2026-04-06 08:08:18,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
258
- INFO 2026-04-06 08:11:11,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
259
- INFO 2026-04-06 08:14:03,148 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
260
- INFO 2026-04-06 08:16:57,119 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
261
- INFO 2026-04-06 08:19:48,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
262
- INFO 2026-04-06 08:22:39,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
263
- INFO 2026-04-06 08:25:31,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
264
- INFO 2026-04-06 08:28:25,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
265
- INFO 2026-04-06 08:31:16,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
266
- INFO 2026-04-06 08:34:09,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
267
- INFO 2026-04-06 08:37:01,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
268
- INFO 2026-04-06 08:39:53,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
269
- INFO 2026-04-06 08:42:45,381 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
270
- INFO 2026-04-06 08:45:38,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
271
- INFO 2026-04-06 08:48:30,988 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
272
- INFO 2026-04-06 08:51:23,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
273
- INFO 2026-04-06 08:54:13,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
274
- INFO 2026-04-06 08:57:05,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
275
- INFO 2026-04-06 08:59:57,948 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
276
- INFO 2026-04-06 09:02:49,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
277
- INFO 2026-04-06 09:05:41,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
278
- INFO 2026-04-06 09:08:33,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
279
- INFO 2026-04-06 09:11:25,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
280
- INFO 2026-04-06 09:14:17,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
281
- INFO 2026-04-06 09:17:08,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
282
- INFO 2026-04-06 09:20:01,791 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
283
- INFO 2026-04-06 09:22:53,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
284
- INFO 2026-04-06 09:25:46,358 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
285
- INFO 2026-04-06 09:28:40,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
286
- INFO 2026-04-06 09:31:33,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
287
- INFO 2026-04-06 09:34:24,731 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
288
- INFO 2026-04-06 09:37:17,501 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
289
- INFO 2026-04-06 09:40:08,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
290
- INFO 2026-04-06 09:43:00,698 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
291
- INFO 2026-04-06 09:45:52,659 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
292
- INFO 2026-04-06 09:48:44,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
293
- INFO 2026-04-06 09:51:36,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
294
- INFO 2026-04-06 09:54:29,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
295
- INFO 2026-04-06 09:57:20,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
296
- INFO 2026-04-06 10:00:22,859 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
297
- INFO 2026-04-06 10:03:48,882 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
298
- INFO 2026-04-06 10:06:52,082 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
299
- INFO 2026-04-06 10:09:59,025 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
300
- INFO 2026-04-06 10:13:05,838 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
301
- INFO 2026-04-06 10:16:11,931 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
302
- INFO 2026-04-06 10:19:15,973 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
303
- INFO 2026-04-06 10:22:24,662 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
304
- INFO 2026-04-06 10:25:33,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
305
- INFO 2026-04-06 10:28:42,169 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
306
- INFO 2026-04-06 10:31:51,513 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
307
- INFO 2026-04-06 10:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
308
- INFO 2026-04-06 10:38:08,605 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
309
- INFO 2026-04-06 10:41:15,592 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
310
- INFO 2026-04-06 11:29:31,541 barrel.train.components.optimizers.optimizer_module init_optimizer rank=3, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
311
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
312
- hyperparams=ConfigDict[typing.Any](
313
- lr=5e-05,
314
- weight_decay=0.0,
315
- ),
316
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
317
- torch_optimizer='torch.optim.AdamW',
318
- ), scheduler=SequentialSchedulerConfig(
319
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
320
- milestones=ConfigList[int](
321
- 500,
322
- ),
323
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
324
- warmup=TorchSchedulerConfig(
325
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
326
- hyperparams=ConfigDict[typing.Any](
327
- start_factor=0.0002,
328
- end_factor=1.0,
329
- total_iters=500,
330
- ),
331
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
332
- ),
333
- const=TorchSchedulerConfig(
334
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
335
- hyperparams=ConfigDict[typing.Any](
336
- factor=1.0,
337
- total_iters=300000,
338
- ),
339
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
340
- ),
341
- ),
342
- )
343
- INFO 2026-04-06 11:29:31,542 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
344
- INFO 2026-04-06 11:29:34,957 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
345
- INFO 2026-04-06 11:29:43,734 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
346
- INFO 2026-04-06 11:29:48,010 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
347
- INFO 2026-04-06 11:29:51,747 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
348
- INFO 2026-04-06 11:29:51,748 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=16790.199999999997, dataset_name=bridge_paraphrase_k5
349
- INFO 2026-04-06 11:29:51,791 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183231, rank=3
350
- INFO 2026-04-06 11:29:52,609 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=3, status=start
351
- INFO 2026-04-06 11:29:52,648 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
352
- INFO 2026-04-06 11:29:52,650 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=21107.399999999998
353
- INFO 2026-04-06 11:29:52,655 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=3, status=start
354
- INFO 2026-04-06 11:31:39,839 barrel.train.core.trainer.trainer Restore session checkpoint rank=3, status=success
355
- INFO 2026-04-06 11:32:01,842 barrel.train.core.trainer.trainer session_init status=success, rank=3
356
- INFO 2026-04-06 11:32:01,844 barrel.train.core.trainer.trainer start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
357
- INFO 2026-04-06 11:32:02,012 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-7, worker_pid=3492781, parent_pid=3489443, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
358
- INFO 2026-04-06 11:32:02,046 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-7, worker_pid=3492823, parent_pid=3489443, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
359
- INFO 2026-04-06 11:35:29,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
360
- INFO 2026-04-06 11:38:35,205 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
361
- INFO 2026-04-06 11:41:41,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
362
- INFO 2026-04-06 11:44:46,853 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
363
- INFO 2026-04-06 11:47:52,316 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
364
- INFO 2026-04-06 11:50:56,954 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
365
- INFO 2026-04-06 11:54:03,028 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
366
- INFO 2026-04-06 11:57:07,427 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
367
- INFO 2026-04-06 12:00:13,670 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
368
- INFO 2026-04-06 12:03:19,356 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
369
- INFO 2026-04-06 12:06:23,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
370
- INFO 2026-04-06 12:09:29,449 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
371
- INFO 2026-04-06 12:12:35,573 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
372
- INFO 2026-04-06 12:15:41,419 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
373
- INFO 2026-04-06 12:18:46,707 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
374
- INFO 2026-04-06 12:21:51,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
375
- INFO 2026-04-06 12:24:57,751 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
376
- INFO 2026-04-06 12:28:01,900 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
377
- INFO 2026-04-06 12:31:07,226 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
378
- INFO 2026-04-06 12:34:11,869 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
379
- INFO 2026-04-06 12:37:16,742 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
380
- INFO 2026-04-06 12:40:21,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
381
- INFO 2026-04-06 12:43:25,439 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
382
- INFO 2026-04-06 12:46:31,794 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
383
- INFO 2026-04-06 12:49:37,509 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
384
- INFO 2026-04-06 12:52:43,066 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
385
- INFO 2026-04-06 12:55:47,773 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
386
- INFO 2026-04-06 12:58:53,491 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
387
- INFO 2026-04-06 13:01:58,054 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
388
- INFO 2026-04-06 13:05:03,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
389
- INFO 2026-04-06 13:08:09,161 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
390
- INFO 2026-04-06 13:11:13,202 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
391
- INFO 2026-04-06 13:14:19,371 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
392
- INFO 2026-04-06 13:17:25,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
393
- INFO 2026-04-06 13:20:30,205 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
394
- INFO 2026-04-06 13:23:35,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
395
- INFO 2026-04-06 13:26:41,883 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
396
- INFO 2026-04-06 13:29:48,085 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
397
- INFO 2026-04-06 13:32:52,621 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
398
- INFO 2026-04-06 13:35:57,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
399
- INFO 2026-04-06 13:39:03,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
400
- INFO 2026-04-06 13:42:10,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
401
- INFO 2026-04-06 13:45:14,642 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
402
- INFO 2026-04-06 13:48:20,973 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
403
- INFO 2026-04-06 13:51:25,870 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
404
- INFO 2026-04-06 13:54:30,723 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
405
- INFO 2026-04-06 13:57:36,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
406
- INFO 2026-04-06 14:00:41,878 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
407
- INFO 2026-04-06 14:03:49,035 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
408
- INFO 2026-04-06 14:07:04,756 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
409
- INFO 2026-04-06 14:10:39,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
410
- INFO 2026-04-06 14:13:44,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
411
- INFO 2026-04-06 14:16:49,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
412
- INFO 2026-04-06 14:19:53,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
413
- INFO 2026-04-06 14:22:58,780 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
414
- INFO 2026-04-06 14:26:03,765 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
415
- INFO 2026-04-06 14:29:07,820 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
416
- INFO 2026-04-06 14:32:11,856 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
417
- INFO 2026-04-06 14:35:16,311 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
418
- INFO 2026-04-06 14:38:22,038 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
419
- INFO 2026-04-06 14:41:25,477 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
420
- INFO 2026-04-06 14:44:30,533 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
421
- INFO 2026-04-06 14:47:36,055 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
422
- INFO 2026-04-06 14:50:40,772 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
423
- INFO 2026-04-06 14:53:45,038 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
424
- INFO 2026-04-06 14:56:50,257 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
425
- INFO 2026-04-06 14:59:56,690 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
426
- INFO 2026-04-06 15:03:02,328 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
427
- INFO 2026-04-07 00:41:05,434 barrel.train.components.optimizers.optimizer_module init_optimizer rank=3, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
428
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
429
- hyperparams=ConfigDict[typing.Any](
430
- lr=5e-05,
431
- weight_decay=0.0,
432
- ),
433
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
434
- torch_optimizer='torch.optim.AdamW',
435
- ), scheduler=SequentialSchedulerConfig(
436
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
437
- milestones=ConfigList[int](
438
- 500,
439
- ),
440
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
441
- warmup=TorchSchedulerConfig(
442
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
443
- hyperparams=ConfigDict[typing.Any](
444
- start_factor=0.0002,
445
- end_factor=1.0,
446
- total_iters=500,
447
- ),
448
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
449
- ),
450
- const=TorchSchedulerConfig(
451
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
452
- hyperparams=ConfigDict[typing.Any](
453
- factor=1.0,
454
- total_iters=300000,
455
- ),
456
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
457
- ),
458
- ),
459
- )
460
- INFO 2026-04-07 00:41:05,438 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
461
- INFO 2026-04-07 00:41:08,565 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
462
- INFO 2026-04-07 00:41:17,673 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
463
- INFO 2026-04-07 00:41:22,588 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
464
- INFO 2026-04-07 00:41:26,711 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
465
- INFO 2026-04-07 00:41:26,711 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=18146.3, dataset_name=bridge_paraphrase_k5
466
- INFO 2026-04-07 00:41:26,750 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183231, rank=3
467
- INFO 2026-04-07 00:41:27,767 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=3, status=start
468
- INFO 2026-04-07 00:41:27,806 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
469
- INFO 2026-04-07 00:41:27,807 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=22369.5
470
- INFO 2026-04-07 00:41:27,812 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=3, status=start
471
- INFO 2026-04-07 00:43:48,102 barrel.train.core.trainer.trainer Restore session checkpoint rank=3, status=success
472
- INFO 2026-04-07 00:44:10,421 barrel.train.core.trainer.trainer session_init status=success, rank=3
473
- INFO 2026-04-07 00:44:10,423 barrel.train.core.trainer.trainer start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
474
- INFO 2026-04-07 00:44:10,492 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=3313508, parent_pid=3310181, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
475
- INFO 2026-04-07 00:44:10,499 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=3313515, parent_pid=3310181, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
476
- INFO 2026-04-07 00:47:36,031 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
477
- INFO 2026-04-07 00:50:38,868 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
478
- INFO 2026-04-07 00:53:42,311 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
479
- INFO 2026-04-07 00:56:45,085 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
480
- INFO 2026-04-07 00:59:47,986 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
481
- INFO 2026-04-07 01:02:50,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
482
- INFO 2026-04-07 01:05:53,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
483
- INFO 2026-04-07 01:08:56,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
484
- INFO 2026-04-07 01:11:58,575 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
485
- INFO 2026-04-07 01:15:02,202 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
486
- INFO 2026-04-07 01:18:05,102 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
487
- INFO 2026-04-07 01:21:08,473 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
488
- INFO 2026-04-07 01:24:11,700 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
489
- INFO 2026-04-07 01:27:15,144 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
490
- INFO 2026-04-07 01:30:16,886 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
491
- INFO 2026-04-07 01:33:19,435 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
492
- INFO 2026-04-07 01:36:22,455 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
493
- INFO 2026-04-07 01:39:25,144 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
494
- INFO 2026-04-07 01:42:29,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
495
- INFO 2026-04-07 01:45:32,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
496
- INFO 2026-04-07 01:48:36,838 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
497
- INFO 2026-04-07 01:51:39,490 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
498
- INFO 2026-04-07 01:54:42,916 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
499
- INFO 2026-04-07 01:57:46,410 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
500
- INFO 2026-04-07 02:00:50,172 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
501
- INFO 2026-04-07 02:03:54,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
502
- INFO 2026-04-07 02:06:57,977 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
503
- INFO 2026-04-07 02:10:02,504 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
504
- INFO 2026-04-07 02:13:05,977 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
505
- INFO 2026-04-07 02:16:10,152 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
506
- INFO 2026-04-07 02:19:14,239 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
507
- INFO 2026-04-07 02:22:17,138 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
508
- INFO 2026-04-07 02:25:21,414 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
509
- INFO 2026-04-07 02:28:24,737 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
510
- INFO 2026-04-07 02:31:29,706 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
511
- INFO 2026-04-07 02:34:33,435 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
512
- INFO 2026-04-07 02:37:37,740 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
513
- INFO 2026-04-07 02:40:41,389 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
514
- INFO 2026-04-07 02:43:44,851 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
515
- INFO 2026-04-07 02:46:48,385 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
516
- INFO 2026-04-07 02:49:52,438 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
517
- INFO 2026-04-07 02:52:56,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
518
- INFO 2026-04-07 02:56:01,910 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
519
- INFO 2026-04-07 02:59:07,494 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
520
- INFO 2026-04-07 03:02:11,723 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
521
- INFO 2026-04-07 03:05:16,729 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
522
- INFO 2026-04-07 03:08:20,100 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
523
- INFO 2026-04-07 03:11:23,981 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
524
- INFO 2026-04-07 03:14:28,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
525
- INFO 2026-04-07 03:17:51,598 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
526
- INFO 2026-04-07 03:18:34,222 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=3
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7576ddc2df7f608e2163d65ac4826d06e45769daa23ea34fe0212c932de2f58
3
+ size 64043
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_4.log CHANGED
@@ -1,526 +1,3 @@
1
- INFO 2026-04-05 07:47:39,879 barrel.train.components.optimizers.optimizer_module init_optimizer rank=4, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
2
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
3
- hyperparams=ConfigDict[typing.Any](
4
- lr=5e-05,
5
- weight_decay=0.0,
6
- ),
7
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
8
- torch_optimizer='torch.optim.AdamW',
9
- ), scheduler=SequentialSchedulerConfig(
10
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
11
- milestones=ConfigList[int](
12
- 500,
13
- ),
14
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
15
- warmup=TorchSchedulerConfig(
16
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
17
- hyperparams=ConfigDict[typing.Any](
18
- start_factor=0.0002,
19
- end_factor=1.0,
20
- total_iters=500,
21
- ),
22
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
23
- ),
24
- const=TorchSchedulerConfig(
25
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
26
- hyperparams=ConfigDict[typing.Any](
27
- factor=1.0,
28
- total_iters=300000,
29
- ),
30
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
31
- ),
32
- ),
33
- )
34
- INFO 2026-04-05 07:47:39,880 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
35
- INFO 2026-04-05 07:47:41,969 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge
36
- INFO 2026-04-05 07:47:55,803 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
37
- INFO 2026-04-05 07:48:00,005 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
38
- INFO 2026-04-05 07:48:00,005 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=18036.0, dataset_name=bridge
39
- INFO 2026-04-05 07:48:00,043 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183262, rank=4
40
- INFO 2026-04-05 07:48:01,112 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge rank=4, status=start
41
- INFO 2026-04-05 07:48:01,151 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
42
- INFO 2026-04-05 07:48:01,152 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=21272.5
43
- INFO 2026-04-05 07:48:34,458 barrel.train.core.trainer.trainer session_init status=success, rank=4
44
- INFO 2026-04-05 07:48:34,460 barrel.train.core.trainer.trainer start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
45
- INFO 2026-04-05 07:48:34,520 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-2, worker_pid=3875751, parent_pid=3872893, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
46
- INFO 2026-04-05 07:48:34,554 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-2, worker_pid=3875773, parent_pid=3872893, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
47
- INFO 2026-04-05 07:51:52,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
48
- INFO 2026-04-05 07:54:44,997 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
49
- INFO 2026-04-05 07:57:35,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
50
- INFO 2026-04-05 08:00:26,862 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
51
- INFO 2026-04-05 08:03:18,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
52
- INFO 2026-04-05 08:06:09,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
53
- INFO 2026-04-05 08:09:01,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
54
- INFO 2026-04-05 08:11:53,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
55
- INFO 2026-04-05 08:14:44,578 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
56
- INFO 2026-04-05 08:17:37,375 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
57
- INFO 2026-04-05 08:20:30,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
58
- INFO 2026-04-05 08:23:23,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
59
- INFO 2026-04-05 08:26:15,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
60
- INFO 2026-04-05 08:29:07,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
61
- INFO 2026-04-05 08:31:59,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
62
- INFO 2026-04-05 08:34:50,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
63
- INFO 2026-04-05 08:37:43,456 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
64
- INFO 2026-04-05 08:40:35,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
65
- INFO 2026-04-05 08:43:28,467 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
66
- INFO 2026-04-05 08:46:21,132 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
67
- INFO 2026-04-05 08:49:13,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
68
- INFO 2026-04-05 08:52:04,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
69
- INFO 2026-04-05 08:54:56,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
70
- INFO 2026-04-05 08:57:48,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
71
- INFO 2026-04-05 09:00:40,238 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
72
- INFO 2026-04-05 09:03:32,325 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
73
- INFO 2026-04-05 09:06:23,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
74
- INFO 2026-04-05 09:09:15,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
75
- INFO 2026-04-05 09:12:08,117 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
76
- INFO 2026-04-05 09:14:59,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
77
- INFO 2026-04-05 09:17:51,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
78
- INFO 2026-04-05 09:20:42,298 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
79
- INFO 2026-04-05 09:23:34,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
80
- INFO 2026-04-05 09:26:26,063 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
81
- INFO 2026-04-05 09:29:18,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
82
- INFO 2026-04-05 09:32:09,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
83
- INFO 2026-04-05 09:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
84
- INFO 2026-04-05 09:37:52,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
85
- INFO 2026-04-05 09:40:43,428 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
86
- INFO 2026-04-05 09:43:34,586 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
87
- INFO 2026-04-05 09:46:26,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
88
- INFO 2026-04-05 09:49:17,267 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
89
- INFO 2026-04-05 09:52:06,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
90
- INFO 2026-04-05 09:54:57,889 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
91
- INFO 2026-04-05 09:57:48,894 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
92
- INFO 2026-04-05 10:00:41,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
93
- INFO 2026-04-05 10:03:33,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
94
- INFO 2026-04-05 10:06:24,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
95
- INFO 2026-04-05 10:09:17,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
96
- INFO 2026-04-05 10:12:18,796 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
97
- INFO 2026-04-05 10:15:33,483 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
98
- INFO 2026-04-05 10:18:24,744 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
99
- INFO 2026-04-05 10:21:17,302 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
100
- INFO 2026-04-05 10:24:08,917 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
101
- INFO 2026-04-05 10:27:00,005 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
102
- INFO 2026-04-05 10:29:51,959 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
103
- INFO 2026-04-05 10:32:42,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
104
- INFO 2026-04-05 10:35:34,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
105
- INFO 2026-04-05 10:38:25,583 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
106
- INFO 2026-04-05 10:41:18,350 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
107
- INFO 2026-04-05 10:44:08,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
108
- INFO 2026-04-05 10:47:00,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
109
- INFO 2026-04-05 10:49:51,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
110
- INFO 2026-04-05 10:52:43,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
111
- INFO 2026-04-05 10:55:35,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
112
- INFO 2026-04-05 10:58:26,949 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
113
- INFO 2026-04-05 11:01:18,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
114
- INFO 2026-04-05 11:04:10,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
115
- INFO 2026-04-05 11:07:02,461 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
116
- INFO 2026-04-05 11:09:53,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
117
- INFO 2026-04-05 11:12:45,451 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
118
- INFO 2026-04-05 11:15:36,492 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
119
- INFO 2026-04-05 11:18:28,400 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
120
- INFO 2026-04-05 11:21:20,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
121
- INFO 2026-04-05 11:24:11,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
122
- INFO 2026-04-05 11:27:02,198 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
123
- INFO 2026-04-05 11:29:53,736 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
124
- INFO 2026-04-05 11:32:44,011 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
125
- INFO 2026-04-05 11:35:35,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
126
- INFO 2026-04-05 11:38:25,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
127
- INFO 2026-04-05 11:41:15,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
128
- INFO 2026-04-05 11:44:05,134 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
129
- INFO 2026-04-05 11:46:56,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
130
- INFO 2026-04-05 11:49:47,469 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
131
- INFO 2026-04-05 11:52:38,563 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
132
- INFO 2026-04-05 11:55:30,289 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
133
- INFO 2026-04-05 11:58:21,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
134
- INFO 2026-04-05 12:01:13,425 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
135
- INFO 2026-04-05 12:04:04,015 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
136
- INFO 2026-04-05 12:06:55,196 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
137
- INFO 2026-04-05 12:09:46,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
138
- INFO 2026-04-05 12:12:37,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
139
- INFO 2026-04-05 12:15:28,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
140
- INFO 2026-04-05 12:18:19,903 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
141
- INFO 2026-04-05 12:21:11,565 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
142
- INFO 2026-04-05 12:24:02,399 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
143
- INFO 2026-04-05 12:26:54,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
144
- INFO 2026-04-05 12:29:45,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
145
- INFO 2026-04-05 12:32:36,286 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
146
- INFO 2026-04-05 12:35:36,892 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
147
- INFO 2026-04-05 12:38:50,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
148
- INFO 2026-04-05 12:41:41,343 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
149
- INFO 2026-04-05 12:44:32,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
150
- INFO 2026-04-05 12:47:23,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
151
- INFO 2026-04-05 12:50:13,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
152
- INFO 2026-04-05 12:53:06,274 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
153
- INFO 2026-04-05 12:55:56,979 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
154
- INFO 2026-04-05 12:58:48,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
155
- INFO 2026-04-05 13:01:39,887 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
156
- INFO 2026-04-05 13:04:31,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
157
- INFO 2026-04-05 13:07:22,318 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
158
- INFO 2026-04-05 13:10:15,398 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
159
- INFO 2026-04-05 13:13:06,519 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
160
- INFO 2026-04-05 13:15:57,164 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
161
- INFO 2026-04-05 13:18:49,647 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
162
- INFO 2026-04-05 13:21:40,162 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
163
- INFO 2026-04-05 13:24:31,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
164
- INFO 2026-04-05 13:27:22,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
165
- INFO 2026-04-05 13:30:13,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
166
- INFO 2026-04-05 13:33:06,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
167
- INFO 2026-04-05 13:35:57,606 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
168
- INFO 2026-04-05 13:38:47,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
169
- INFO 2026-04-05 13:41:40,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
170
- INFO 2026-04-05 13:44:31,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
171
- INFO 2026-04-05 13:47:23,254 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
172
- INFO 2026-04-05 13:50:14,660 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
173
- INFO 2026-04-05 13:53:06,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
174
- INFO 2026-04-05 13:55:58,107 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
175
- INFO 2026-04-05 13:58:50,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
176
- INFO 2026-04-05 14:01:41,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
177
- INFO 2026-04-05 14:04:32,278 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
178
- INFO 2026-04-05 14:07:24,300 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
179
- INFO 2026-04-05 14:10:15,785 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
180
- INFO 2026-04-05 14:13:07,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
181
- INFO 2026-04-05 14:15:58,762 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
182
- INFO 2026-04-05 14:18:49,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
183
- INFO 2026-04-05 14:21:40,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
184
- INFO 2026-04-05 14:24:31,803 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
185
- INFO 2026-04-05 14:27:22,607 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
186
- INFO 2026-04-05 14:30:14,379 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
187
- INFO 2026-04-05 14:33:06,049 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
188
- INFO 2026-04-05 14:35:56,094 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
189
- INFO 2026-04-05 14:38:48,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
190
- INFO 2026-04-05 14:41:40,352 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
191
- INFO 2026-04-05 14:44:32,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
192
- INFO 2026-04-05 14:47:25,214 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
193
- INFO 2026-04-05 14:50:15,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
194
- INFO 2026-04-05 14:53:07,348 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
195
- INFO 2026-04-05 14:56:00,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
196
- INFO 2026-04-05 14:59:02,160 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
197
- INFO 2026-04-05 14:59:35,376 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=4
198
- INFO 2026-04-06 07:33:53,235 barrel.train.components.optimizers.optimizer_module init_optimizer rank=4, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
199
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
200
- hyperparams=ConfigDict[typing.Any](
201
- lr=5e-05,
202
- weight_decay=0.0,
203
- ),
204
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
205
- torch_optimizer='torch.optim.AdamW',
206
- ), scheduler=SequentialSchedulerConfig(
207
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
208
- milestones=ConfigList[int](
209
- 500,
210
- ),
211
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
212
- warmup=TorchSchedulerConfig(
213
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
214
- hyperparams=ConfigDict[typing.Any](
215
- start_factor=0.0002,
216
- end_factor=1.0,
217
- total_iters=500,
218
- ),
219
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
220
- ),
221
- const=TorchSchedulerConfig(
222
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
223
- hyperparams=ConfigDict[typing.Any](
224
- factor=1.0,
225
- total_iters=300000,
226
- ),
227
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
228
- ),
229
- ),
230
- )
231
- INFO 2026-04-06 07:33:53,239 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
232
- INFO 2026-04-06 07:33:56,085 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
233
- INFO 2026-04-06 07:34:05,279 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
234
- INFO 2026-04-06 07:34:10,089 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
235
- INFO 2026-04-06 07:34:14,165 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
236
- INFO 2026-04-06 07:34:14,165 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=18080.2, dataset_name=bridge_paraphrase_k5
237
- INFO 2026-04-06 07:34:14,205 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183262, rank=4
238
- INFO 2026-04-06 07:34:15,165 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=4, status=start
239
- INFO 2026-04-06 07:34:15,205 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
240
- INFO 2026-04-06 07:34:15,206 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=21967.600000000002
241
- INFO 2026-04-06 07:34:15,212 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=4, status=start
242
- INFO 2026-04-06 07:36:05,814 barrel.train.core.trainer.trainer Restore session checkpoint rank=4, status=success
243
- INFO 2026-04-06 07:36:27,876 barrel.train.core.trainer.trainer session_init status=success, rank=4
244
- INFO 2026-04-06 07:36:27,879 barrel.train.core.trainer.trainer start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
245
- INFO 2026-04-06 07:36:27,923 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=2465167, parent_pid=2461889, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
246
- INFO 2026-04-06 07:36:27,947 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=2465176, parent_pid=2461889, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
247
- INFO 2026-04-06 07:39:43,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
248
- INFO 2026-04-06 07:42:35,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
249
- INFO 2026-04-06 07:45:26,710 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
250
- INFO 2026-04-06 07:48:18,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
251
- INFO 2026-04-06 07:51:08,587 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
252
- INFO 2026-04-06 07:54:00,463 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
253
- INFO 2026-04-06 07:56:51,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
254
- INFO 2026-04-06 07:59:43,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
255
- INFO 2026-04-06 08:02:34,655 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
256
- INFO 2026-04-06 08:05:26,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
257
- INFO 2026-04-06 08:08:18,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
258
- INFO 2026-04-06 08:11:11,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
259
- INFO 2026-04-06 08:14:03,148 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
260
- INFO 2026-04-06 08:16:57,119 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
261
- INFO 2026-04-06 08:19:48,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
262
- INFO 2026-04-06 08:22:39,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
263
- INFO 2026-04-06 08:25:31,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
264
- INFO 2026-04-06 08:28:25,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
265
- INFO 2026-04-06 08:31:16,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
266
- INFO 2026-04-06 08:34:09,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
267
- INFO 2026-04-06 08:37:01,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
268
- INFO 2026-04-06 08:39:53,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
269
- INFO 2026-04-06 08:42:45,381 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
270
- INFO 2026-04-06 08:45:38,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
271
- INFO 2026-04-06 08:48:30,988 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
272
- INFO 2026-04-06 08:51:23,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
273
- INFO 2026-04-06 08:54:13,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
274
- INFO 2026-04-06 08:57:05,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
275
- INFO 2026-04-06 08:59:57,948 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
276
- INFO 2026-04-06 09:02:49,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
277
- INFO 2026-04-06 09:05:41,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
278
- INFO 2026-04-06 09:08:33,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
279
- INFO 2026-04-06 09:11:25,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
280
- INFO 2026-04-06 09:14:17,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
281
- INFO 2026-04-06 09:17:08,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
282
- INFO 2026-04-06 09:20:01,791 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
283
- INFO 2026-04-06 09:22:53,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
284
- INFO 2026-04-06 09:25:46,358 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
285
- INFO 2026-04-06 09:28:40,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
286
- INFO 2026-04-06 09:31:33,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
287
- INFO 2026-04-06 09:34:24,731 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
288
- INFO 2026-04-06 09:37:17,501 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
289
- INFO 2026-04-06 09:40:08,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
290
- INFO 2026-04-06 09:43:00,698 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
291
- INFO 2026-04-06 09:45:52,659 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
292
- INFO 2026-04-06 09:48:44,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
293
- INFO 2026-04-06 09:51:36,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
294
- INFO 2026-04-06 09:54:29,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
295
- INFO 2026-04-06 09:57:20,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
296
- INFO 2026-04-06 10:00:22,725 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
297
- INFO 2026-04-06 10:03:48,884 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
298
- INFO 2026-04-06 10:06:52,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
299
- INFO 2026-04-06 10:09:59,025 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
300
- INFO 2026-04-06 10:13:05,840 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
301
- INFO 2026-04-06 10:16:11,933 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
302
- INFO 2026-04-06 10:19:15,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
303
- INFO 2026-04-06 10:22:24,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
304
- INFO 2026-04-06 10:25:33,536 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
305
- INFO 2026-04-06 10:28:42,166 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
306
- INFO 2026-04-06 10:31:51,506 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
307
- INFO 2026-04-06 10:35:00,438 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
308
- INFO 2026-04-06 10:38:08,601 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
309
- INFO 2026-04-06 10:41:15,593 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
310
- INFO 2026-04-06 11:29:31,565 barrel.train.components.optimizers.optimizer_module init_optimizer rank=4, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
311
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
312
- hyperparams=ConfigDict[typing.Any](
313
- lr=5e-05,
314
- weight_decay=0.0,
315
- ),
316
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
317
- torch_optimizer='torch.optim.AdamW',
318
- ), scheduler=SequentialSchedulerConfig(
319
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
320
- milestones=ConfigList[int](
321
- 500,
322
- ),
323
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
324
- warmup=TorchSchedulerConfig(
325
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
326
- hyperparams=ConfigDict[typing.Any](
327
- start_factor=0.0002,
328
- end_factor=1.0,
329
- total_iters=500,
330
- ),
331
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
332
- ),
333
- const=TorchSchedulerConfig(
334
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
335
- hyperparams=ConfigDict[typing.Any](
336
- factor=1.0,
337
- total_iters=300000,
338
- ),
339
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
340
- ),
341
- ),
342
- )
343
- INFO 2026-04-06 11:29:31,569 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
344
- INFO 2026-04-06 11:29:34,896 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
345
- INFO 2026-04-06 11:29:43,630 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
346
- INFO 2026-04-06 11:29:47,749 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
347
- INFO 2026-04-06 11:29:51,261 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
348
- INFO 2026-04-06 11:29:51,262 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=16366.199999999999, dataset_name=bridge_paraphrase_k5
349
- INFO 2026-04-06 11:29:51,302 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183262, rank=4
350
- INFO 2026-04-06 11:29:52,100 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=4, status=start
351
- INFO 2026-04-06 11:29:52,139 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
352
- INFO 2026-04-06 11:29:52,142 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=20573.3
353
- INFO 2026-04-06 11:29:52,149 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=4, status=start
354
- INFO 2026-04-06 11:31:39,841 barrel.train.core.trainer.trainer Restore session checkpoint rank=4, status=success
355
- INFO 2026-04-06 11:32:01,842 barrel.train.core.trainer.trainer session_init status=success, rank=4
356
- INFO 2026-04-06 11:32:01,844 barrel.train.core.trainer.trainer start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
357
- INFO 2026-04-06 11:32:02,007 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-7, worker_pid=3492779, parent_pid=3489444, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
358
- INFO 2026-04-06 11:32:02,034 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-7, worker_pid=3492797, parent_pid=3489444, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
359
- INFO 2026-04-06 11:35:29,335 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
360
- INFO 2026-04-06 11:38:35,205 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
361
- INFO 2026-04-06 11:41:41,429 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
362
- INFO 2026-04-06 11:44:46,850 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
363
- INFO 2026-04-06 11:47:52,316 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
364
- INFO 2026-04-06 11:50:56,955 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
365
- INFO 2026-04-06 11:54:03,030 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
366
- INFO 2026-04-06 11:57:07,425 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
367
- INFO 2026-04-06 12:00:13,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
368
- INFO 2026-04-06 12:03:19,354 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
369
- INFO 2026-04-06 12:06:23,698 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
370
- INFO 2026-04-06 12:09:29,449 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
371
- INFO 2026-04-06 12:12:35,571 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
372
- INFO 2026-04-06 12:15:41,421 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
373
- INFO 2026-04-06 12:18:46,705 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
374
- INFO 2026-04-06 12:21:51,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
375
- INFO 2026-04-06 12:24:57,748 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
376
- INFO 2026-04-06 12:28:01,900 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
377
- INFO 2026-04-06 12:31:07,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
378
- INFO 2026-04-06 12:34:11,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
379
- INFO 2026-04-06 12:37:16,741 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
380
- INFO 2026-04-06 12:40:21,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
381
- INFO 2026-04-06 12:43:25,437 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
382
- INFO 2026-04-06 12:46:31,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
383
- INFO 2026-04-06 12:49:37,508 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
384
- INFO 2026-04-06 12:52:43,064 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
385
- INFO 2026-04-06 12:55:47,771 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
386
- INFO 2026-04-06 12:58:53,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
387
- INFO 2026-04-06 13:01:58,052 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
388
- INFO 2026-04-06 13:05:03,713 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
389
- INFO 2026-04-06 13:08:09,156 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
390
- INFO 2026-04-06 13:11:13,197 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
391
- INFO 2026-04-06 13:14:19,370 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
392
- INFO 2026-04-06 13:17:25,221 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
393
- INFO 2026-04-06 13:20:30,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
394
- INFO 2026-04-06 13:23:35,856 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
395
- INFO 2026-04-06 13:26:41,882 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
396
- INFO 2026-04-06 13:29:48,082 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
397
- INFO 2026-04-06 13:32:52,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
398
- INFO 2026-04-06 13:35:57,630 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
399
- INFO 2026-04-06 13:39:03,246 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
400
- INFO 2026-04-06 13:42:10,407 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
401
- INFO 2026-04-06 13:45:14,630 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
402
- INFO 2026-04-06 13:48:20,969 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
403
- INFO 2026-04-06 13:51:25,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
404
- INFO 2026-04-06 13:54:30,723 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
405
- INFO 2026-04-06 13:57:36,004 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
406
- INFO 2026-04-06 14:00:41,875 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
407
- INFO 2026-04-06 14:03:49,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
408
- INFO 2026-04-06 14:07:04,677 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
409
- INFO 2026-04-06 14:10:39,271 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
410
- INFO 2026-04-06 14:13:44,081 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
411
- INFO 2026-04-06 14:16:49,285 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
412
- INFO 2026-04-06 14:19:53,530 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
413
- INFO 2026-04-06 14:22:58,779 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
414
- INFO 2026-04-06 14:26:03,764 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
415
- INFO 2026-04-06 14:29:07,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
416
- INFO 2026-04-06 14:32:11,857 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
417
- INFO 2026-04-06 14:35:16,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
418
- INFO 2026-04-06 14:38:22,035 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
419
- INFO 2026-04-06 14:41:25,477 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
420
- INFO 2026-04-06 14:44:30,531 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
421
- INFO 2026-04-06 14:47:36,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
422
- INFO 2026-04-06 14:50:40,768 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
423
- INFO 2026-04-06 14:53:45,036 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
424
- INFO 2026-04-06 14:56:50,255 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
425
- INFO 2026-04-06 14:59:56,689 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
426
- INFO 2026-04-06 15:03:02,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
427
- INFO 2026-04-07 00:41:05,401 barrel.train.components.optimizers.optimizer_module init_optimizer rank=4, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
428
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
429
- hyperparams=ConfigDict[typing.Any](
430
- lr=5e-05,
431
- weight_decay=0.0,
432
- ),
433
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
434
- torch_optimizer='torch.optim.AdamW',
435
- ), scheduler=SequentialSchedulerConfig(
436
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
437
- milestones=ConfigList[int](
438
- 500,
439
- ),
440
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
441
- warmup=TorchSchedulerConfig(
442
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
443
- hyperparams=ConfigDict[typing.Any](
444
- start_factor=0.0002,
445
- end_factor=1.0,
446
- total_iters=500,
447
- ),
448
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
449
- ),
450
- const=TorchSchedulerConfig(
451
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
452
- hyperparams=ConfigDict[typing.Any](
453
- factor=1.0,
454
- total_iters=300000,
455
- ),
456
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
457
- ),
458
- ),
459
- )
460
- INFO 2026-04-07 00:41:05,404 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
461
- INFO 2026-04-07 00:41:08,947 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
462
- INFO 2026-04-07 00:41:26,093 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
463
- INFO 2026-04-07 00:41:36,337 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
464
- INFO 2026-04-07 00:41:40,717 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
465
- INFO 2026-04-07 00:41:40,718 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=31770.600000000002, dataset_name=bridge_paraphrase_k5
466
- INFO 2026-04-07 00:41:40,759 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183262, rank=4
467
- INFO 2026-04-07 00:41:41,580 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=4, status=start
468
- INFO 2026-04-07 00:41:41,621 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
469
- INFO 2026-04-07 00:41:41,624 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=36220.5
470
- INFO 2026-04-07 00:41:41,631 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=4, status=start
471
- INFO 2026-04-07 00:43:48,421 barrel.train.core.trainer.trainer Restore session checkpoint rank=4, status=success
472
- INFO 2026-04-07 00:44:10,422 barrel.train.core.trainer.trainer session_init status=success, rank=4
473
- INFO 2026-04-07 00:44:10,423 barrel.train.core.trainer.trainer start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
474
- INFO 2026-04-07 00:44:10,500 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=3313512, parent_pid=3310182, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
475
- INFO 2026-04-07 00:44:10,529 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=3313543, parent_pid=3310182, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
476
- INFO 2026-04-07 00:47:36,030 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
477
- INFO 2026-04-07 00:50:38,869 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
478
- INFO 2026-04-07 00:53:42,313 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
479
- INFO 2026-04-07 00:56:45,084 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
480
- INFO 2026-04-07 00:59:47,986 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
481
- INFO 2026-04-07 01:02:50,207 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
482
- INFO 2026-04-07 01:05:53,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
483
- INFO 2026-04-07 01:08:56,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
484
- INFO 2026-04-07 01:11:58,573 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
485
- INFO 2026-04-07 01:15:02,204 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
486
- INFO 2026-04-07 01:18:05,102 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
487
- INFO 2026-04-07 01:21:08,471 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
488
- INFO 2026-04-07 01:24:11,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
489
- INFO 2026-04-07 01:27:15,142 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
490
- INFO 2026-04-07 01:30:16,884 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
491
- INFO 2026-04-07 01:33:19,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
492
- INFO 2026-04-07 01:36:22,455 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
493
- INFO 2026-04-07 01:39:25,143 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
494
- INFO 2026-04-07 01:42:29,625 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
495
- INFO 2026-04-07 01:45:32,643 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
496
- INFO 2026-04-07 01:48:36,837 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
497
- INFO 2026-04-07 01:51:39,505 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
498
- INFO 2026-04-07 01:54:42,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
499
- INFO 2026-04-07 01:57:46,408 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
500
- INFO 2026-04-07 02:00:50,171 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
501
- INFO 2026-04-07 02:03:54,280 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
502
- INFO 2026-04-07 02:06:57,975 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
503
- INFO 2026-04-07 02:10:02,505 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
504
- INFO 2026-04-07 02:13:05,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
505
- INFO 2026-04-07 02:16:10,143 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
506
- INFO 2026-04-07 02:19:14,256 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
507
- INFO 2026-04-07 02:22:17,144 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
508
- INFO 2026-04-07 02:25:21,415 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
509
- INFO 2026-04-07 02:28:24,737 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
510
- INFO 2026-04-07 02:31:29,708 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
511
- INFO 2026-04-07 02:34:33,435 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
512
- INFO 2026-04-07 02:37:37,740 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
513
- INFO 2026-04-07 02:40:41,389 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
514
- INFO 2026-04-07 02:43:44,852 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
515
- INFO 2026-04-07 02:46:48,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
516
- INFO 2026-04-07 02:49:52,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
517
- INFO 2026-04-07 02:52:56,832 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
518
- INFO 2026-04-07 02:56:01,912 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
519
- INFO 2026-04-07 02:59:07,336 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
520
- INFO 2026-04-07 03:02:11,725 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
521
- INFO 2026-04-07 03:05:16,728 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
522
- INFO 2026-04-07 03:08:20,110 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
523
- INFO 2026-04-07 03:11:23,978 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
524
- INFO 2026-04-07 03:14:28,687 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
525
- INFO 2026-04-07 03:17:51,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
526
- INFO 2026-04-07 03:18:34,222 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=4
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00983e38d20ab3f03bff50fd7d395420a06294051a5e90ce02b7e6ac04513f4f
3
+ size 64032
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_5.log CHANGED
@@ -1,526 +1,3 @@
1
- INFO 2026-04-05 07:47:39,930 barrel.train.components.optimizers.optimizer_module init_optimizer rank=5, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
2
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
3
- hyperparams=ConfigDict[typing.Any](
4
- lr=5e-05,
5
- weight_decay=0.0,
6
- ),
7
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
8
- torch_optimizer='torch.optim.AdamW',
9
- ), scheduler=SequentialSchedulerConfig(
10
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
11
- milestones=ConfigList[int](
12
- 500,
13
- ),
14
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
15
- warmup=TorchSchedulerConfig(
16
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
17
- hyperparams=ConfigDict[typing.Any](
18
- start_factor=0.0002,
19
- end_factor=1.0,
20
- total_iters=500,
21
- ),
22
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
23
- ),
24
- const=TorchSchedulerConfig(
25
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
26
- hyperparams=ConfigDict[typing.Any](
27
- factor=1.0,
28
- total_iters=300000,
29
- ),
30
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
31
- ),
32
- ),
33
- )
34
- INFO 2026-04-05 07:47:39,931 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
35
- INFO 2026-04-05 07:47:41,994 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge
36
- INFO 2026-04-05 07:48:03,608 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
37
- INFO 2026-04-05 07:48:07,878 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
38
- INFO 2026-04-05 07:48:07,878 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=25884.3, dataset_name=bridge
39
- INFO 2026-04-05 07:48:07,919 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183239, rank=5
40
- INFO 2026-04-05 07:48:08,795 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge rank=5, status=start
41
- INFO 2026-04-05 07:48:08,835 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
42
- INFO 2026-04-05 07:48:08,837 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=28905.6
43
- INFO 2026-04-05 07:48:34,458 barrel.train.core.trainer.trainer session_init status=success, rank=5
44
- INFO 2026-04-05 07:48:34,460 barrel.train.core.trainer.trainer start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
45
- INFO 2026-04-05 07:48:34,514 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-2, worker_pid=3875747, parent_pid=3872894, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
46
- INFO 2026-04-05 07:48:34,535 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-2, worker_pid=3875754, parent_pid=3872894, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
47
- INFO 2026-04-05 07:51:52,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
48
- INFO 2026-04-05 07:54:44,997 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
49
- INFO 2026-04-05 07:57:35,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
50
- INFO 2026-04-05 08:00:26,862 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
51
- INFO 2026-04-05 08:03:18,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
52
- INFO 2026-04-05 08:06:09,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
53
- INFO 2026-04-05 08:09:01,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
54
- INFO 2026-04-05 08:11:53,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
55
- INFO 2026-04-05 08:14:44,578 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
56
- INFO 2026-04-05 08:17:37,375 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
57
- INFO 2026-04-05 08:20:30,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
58
- INFO 2026-04-05 08:23:23,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
59
- INFO 2026-04-05 08:26:15,038 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
60
- INFO 2026-04-05 08:29:07,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
61
- INFO 2026-04-05 08:31:59,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
62
- INFO 2026-04-05 08:34:50,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
63
- INFO 2026-04-05 08:37:43,456 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
64
- INFO 2026-04-05 08:40:35,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
65
- INFO 2026-04-05 08:43:28,467 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
66
- INFO 2026-04-05 08:46:21,132 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
67
- INFO 2026-04-05 08:49:13,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
68
- INFO 2026-04-05 08:52:04,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
69
- INFO 2026-04-05 08:54:56,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
70
- INFO 2026-04-05 08:57:48,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
71
- INFO 2026-04-05 09:00:40,238 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
72
- INFO 2026-04-05 09:03:32,325 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
73
- INFO 2026-04-05 09:06:23,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
74
- INFO 2026-04-05 09:09:15,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
75
- INFO 2026-04-05 09:12:08,116 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
76
- INFO 2026-04-05 09:14:59,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
77
- INFO 2026-04-05 09:17:51,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
78
- INFO 2026-04-05 09:20:42,298 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
79
- INFO 2026-04-05 09:23:34,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
80
- INFO 2026-04-05 09:26:26,064 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
81
- INFO 2026-04-05 09:29:18,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
82
- INFO 2026-04-05 09:32:09,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
83
- INFO 2026-04-05 09:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
84
- INFO 2026-04-05 09:37:52,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
85
- INFO 2026-04-05 09:40:43,428 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
86
- INFO 2026-04-05 09:43:34,586 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
87
- INFO 2026-04-05 09:46:26,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
88
- INFO 2026-04-05 09:49:17,267 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
89
- INFO 2026-04-05 09:52:06,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
90
- INFO 2026-04-05 09:54:57,889 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
91
- INFO 2026-04-05 09:57:48,894 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
92
- INFO 2026-04-05 10:00:41,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
93
- INFO 2026-04-05 10:03:33,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
94
- INFO 2026-04-05 10:06:24,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
95
- INFO 2026-04-05 10:09:17,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
96
- INFO 2026-04-05 10:12:18,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
97
- INFO 2026-04-05 10:15:33,483 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
98
- INFO 2026-04-05 10:18:24,744 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
99
- INFO 2026-04-05 10:21:17,302 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
100
- INFO 2026-04-05 10:24:08,917 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
101
- INFO 2026-04-05 10:27:00,005 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
102
- INFO 2026-04-05 10:29:51,958 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
103
- INFO 2026-04-05 10:32:42,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
104
- INFO 2026-04-05 10:35:34,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
105
- INFO 2026-04-05 10:38:25,583 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
106
- INFO 2026-04-05 10:41:18,350 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
107
- INFO 2026-04-05 10:44:08,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
108
- INFO 2026-04-05 10:47:00,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
109
- INFO 2026-04-05 10:49:51,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
110
- INFO 2026-04-05 10:52:43,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
111
- INFO 2026-04-05 10:55:35,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
112
- INFO 2026-04-05 10:58:26,949 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
113
- INFO 2026-04-05 11:01:18,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
114
- INFO 2026-04-05 11:04:10,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
115
- INFO 2026-04-05 11:07:02,461 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
116
- INFO 2026-04-05 11:09:53,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
117
- INFO 2026-04-05 11:12:45,451 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
118
- INFO 2026-04-05 11:15:36,492 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
119
- INFO 2026-04-05 11:18:28,400 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
120
- INFO 2026-04-05 11:21:20,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
121
- INFO 2026-04-05 11:24:11,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
122
- INFO 2026-04-05 11:27:02,198 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
123
- INFO 2026-04-05 11:29:53,736 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
124
- INFO 2026-04-05 11:32:44,011 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
125
- INFO 2026-04-05 11:35:35,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
126
- INFO 2026-04-05 11:38:25,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
127
- INFO 2026-04-05 11:41:15,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
128
- INFO 2026-04-05 11:44:05,134 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
129
- INFO 2026-04-05 11:46:56,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
130
- INFO 2026-04-05 11:49:47,469 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
131
- INFO 2026-04-05 11:52:38,563 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
132
- INFO 2026-04-05 11:55:30,289 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
133
- INFO 2026-04-05 11:58:21,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
134
- INFO 2026-04-05 12:01:13,424 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
135
- INFO 2026-04-05 12:04:04,015 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
136
- INFO 2026-04-05 12:06:55,196 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
137
- INFO 2026-04-05 12:09:46,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
138
- INFO 2026-04-05 12:12:37,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
139
- INFO 2026-04-05 12:15:28,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
140
- INFO 2026-04-05 12:18:19,903 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
141
- INFO 2026-04-05 12:21:11,565 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
142
- INFO 2026-04-05 12:24:02,399 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
143
- INFO 2026-04-05 12:26:54,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
144
- INFO 2026-04-05 12:29:45,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
145
- INFO 2026-04-05 12:32:36,286 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
146
- INFO 2026-04-05 12:35:36,962 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
147
- INFO 2026-04-05 12:38:50,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
148
- INFO 2026-04-05 12:41:41,343 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
149
- INFO 2026-04-05 12:44:32,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
150
- INFO 2026-04-05 12:47:23,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
151
- INFO 2026-04-05 12:50:13,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
152
- INFO 2026-04-05 12:53:06,274 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
153
- INFO 2026-04-05 12:55:56,979 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
154
- INFO 2026-04-05 12:58:48,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
155
- INFO 2026-04-05 13:01:39,887 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
156
- INFO 2026-04-05 13:04:31,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
157
- INFO 2026-04-05 13:07:22,319 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
158
- INFO 2026-04-05 13:10:15,398 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
159
- INFO 2026-04-05 13:13:06,519 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
160
- INFO 2026-04-05 13:15:57,164 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
161
- INFO 2026-04-05 13:18:49,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
162
- INFO 2026-04-05 13:21:40,162 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
163
- INFO 2026-04-05 13:24:31,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
164
- INFO 2026-04-05 13:27:22,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
165
- INFO 2026-04-05 13:30:13,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
166
- INFO 2026-04-05 13:33:06,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
167
- INFO 2026-04-05 13:35:57,606 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
168
- INFO 2026-04-05 13:38:47,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
169
- INFO 2026-04-05 13:41:40,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
170
- INFO 2026-04-05 13:44:31,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
171
- INFO 2026-04-05 13:47:23,254 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
172
- INFO 2026-04-05 13:50:14,660 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
173
- INFO 2026-04-05 13:53:06,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
174
- INFO 2026-04-05 13:55:58,107 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
175
- INFO 2026-04-05 13:58:50,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
176
- INFO 2026-04-05 14:01:41,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
177
- INFO 2026-04-05 14:04:32,278 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
178
- INFO 2026-04-05 14:07:24,300 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
179
- INFO 2026-04-05 14:10:15,785 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
180
- INFO 2026-04-05 14:13:07,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
181
- INFO 2026-04-05 14:15:58,762 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
182
- INFO 2026-04-05 14:18:49,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
183
- INFO 2026-04-05 14:21:40,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
184
- INFO 2026-04-05 14:24:31,803 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
185
- INFO 2026-04-05 14:27:22,607 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
186
- INFO 2026-04-05 14:30:14,379 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
187
- INFO 2026-04-05 14:33:06,049 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
188
- INFO 2026-04-05 14:35:56,094 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
189
- INFO 2026-04-05 14:38:48,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
190
- INFO 2026-04-05 14:41:40,352 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
191
- INFO 2026-04-05 14:44:32,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
192
- INFO 2026-04-05 14:47:25,214 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
193
- INFO 2026-04-05 14:50:15,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
194
- INFO 2026-04-05 14:53:07,347 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
195
- INFO 2026-04-05 14:56:00,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
196
- INFO 2026-04-05 14:59:02,157 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
197
- INFO 2026-04-05 14:59:35,376 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=5
198
- INFO 2026-04-06 07:33:53,199 barrel.train.components.optimizers.optimizer_module init_optimizer rank=5, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
199
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
200
- hyperparams=ConfigDict[typing.Any](
201
- lr=5e-05,
202
- weight_decay=0.0,
203
- ),
204
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
205
- torch_optimizer='torch.optim.AdamW',
206
- ), scheduler=SequentialSchedulerConfig(
207
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
208
- milestones=ConfigList[int](
209
- 500,
210
- ),
211
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
212
- warmup=TorchSchedulerConfig(
213
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
214
- hyperparams=ConfigDict[typing.Any](
215
- start_factor=0.0002,
216
- end_factor=1.0,
217
- total_iters=500,
218
- ),
219
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
220
- ),
221
- const=TorchSchedulerConfig(
222
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
223
- hyperparams=ConfigDict[typing.Any](
224
- factor=1.0,
225
- total_iters=300000,
226
- ),
227
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
228
- ),
229
- ),
230
- )
231
- INFO 2026-04-06 07:33:53,202 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
232
- INFO 2026-04-06 07:33:56,070 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
233
- INFO 2026-04-06 07:34:05,170 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
234
- INFO 2026-04-06 07:34:09,978 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
235
- INFO 2026-04-06 07:34:13,971 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
236
- INFO 2026-04-06 07:34:13,971 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=17900.7, dataset_name=bridge_paraphrase_k5
237
- INFO 2026-04-06 07:34:14,010 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183239, rank=5
238
- INFO 2026-04-06 07:34:14,982 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=5, status=start
239
- INFO 2026-04-06 07:34:15,021 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
240
- INFO 2026-04-06 07:34:15,022 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=21820.3
241
- INFO 2026-04-06 07:34:15,027 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=5, status=start
242
- INFO 2026-04-06 07:36:05,815 barrel.train.core.trainer.trainer Restore session checkpoint rank=5, status=success
243
- INFO 2026-04-06 07:36:27,876 barrel.train.core.trainer.trainer session_init status=success, rank=5
244
- INFO 2026-04-06 07:36:27,879 barrel.train.core.trainer.trainer start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
245
- INFO 2026-04-06 07:36:27,923 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=2465166, parent_pid=2461890, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
246
- INFO 2026-04-06 07:36:27,945 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=2465172, parent_pid=2461890, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
247
- INFO 2026-04-06 07:39:43,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
248
- INFO 2026-04-06 07:42:35,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
249
- INFO 2026-04-06 07:45:26,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
250
- INFO 2026-04-06 07:48:18,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
251
- INFO 2026-04-06 07:51:08,588 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
252
- INFO 2026-04-06 07:54:00,463 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
253
- INFO 2026-04-06 07:56:51,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
254
- INFO 2026-04-06 07:59:43,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
255
- INFO 2026-04-06 08:02:34,656 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
256
- INFO 2026-04-06 08:05:26,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
257
- INFO 2026-04-06 08:08:18,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
258
- INFO 2026-04-06 08:11:11,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
259
- INFO 2026-04-06 08:14:03,148 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
260
- INFO 2026-04-06 08:16:57,119 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
261
- INFO 2026-04-06 08:19:48,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
262
- INFO 2026-04-06 08:22:39,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
263
- INFO 2026-04-06 08:25:31,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
264
- INFO 2026-04-06 08:28:25,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
265
- INFO 2026-04-06 08:31:16,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
266
- INFO 2026-04-06 08:34:09,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
267
- INFO 2026-04-06 08:37:01,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
268
- INFO 2026-04-06 08:39:53,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
269
- INFO 2026-04-06 08:42:45,381 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
270
- INFO 2026-04-06 08:45:38,330 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
271
- INFO 2026-04-06 08:48:30,989 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
272
- INFO 2026-04-06 08:51:23,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
273
- INFO 2026-04-06 08:54:13,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
274
- INFO 2026-04-06 08:57:05,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
275
- INFO 2026-04-06 08:59:57,948 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
276
- INFO 2026-04-06 09:02:49,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
277
- INFO 2026-04-06 09:05:41,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
278
- INFO 2026-04-06 09:08:33,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
279
- INFO 2026-04-06 09:11:25,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
280
- INFO 2026-04-06 09:14:17,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
281
- INFO 2026-04-06 09:17:08,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
282
- INFO 2026-04-06 09:20:01,791 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
283
- INFO 2026-04-06 09:22:53,270 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
284
- INFO 2026-04-06 09:25:46,358 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
285
- INFO 2026-04-06 09:28:40,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
286
- INFO 2026-04-06 09:31:33,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
287
- INFO 2026-04-06 09:34:24,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
288
- INFO 2026-04-06 09:37:17,501 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
289
- INFO 2026-04-06 09:40:08,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
290
- INFO 2026-04-06 09:43:00,698 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
291
- INFO 2026-04-06 09:45:52,659 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
292
- INFO 2026-04-06 09:48:44,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
293
- INFO 2026-04-06 09:51:36,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
294
- INFO 2026-04-06 09:54:29,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
295
- INFO 2026-04-06 09:57:20,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
296
- INFO 2026-04-06 10:00:22,924 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
297
- INFO 2026-04-06 10:03:48,881 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
298
- INFO 2026-04-06 10:06:52,069 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
299
- INFO 2026-04-06 10:09:59,021 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
300
- INFO 2026-04-06 10:13:05,836 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
301
- INFO 2026-04-06 10:16:11,928 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
302
- INFO 2026-04-06 10:19:15,972 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
303
- INFO 2026-04-06 10:22:24,650 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
304
- INFO 2026-04-06 10:25:33,535 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
305
- INFO 2026-04-06 10:28:42,167 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
306
- INFO 2026-04-06 10:31:51,504 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
307
- INFO 2026-04-06 10:35:00,432 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
308
- INFO 2026-04-06 10:38:08,599 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
309
- INFO 2026-04-06 10:41:15,591 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
310
- INFO 2026-04-06 11:29:31,543 barrel.train.components.optimizers.optimizer_module init_optimizer rank=5, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
311
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
312
- hyperparams=ConfigDict[typing.Any](
313
- lr=5e-05,
314
- weight_decay=0.0,
315
- ),
316
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
317
- torch_optimizer='torch.optim.AdamW',
318
- ), scheduler=SequentialSchedulerConfig(
319
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
320
- milestones=ConfigList[int](
321
- 500,
322
- ),
323
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
324
- warmup=TorchSchedulerConfig(
325
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
326
- hyperparams=ConfigDict[typing.Any](
327
- start_factor=0.0002,
328
- end_factor=1.0,
329
- total_iters=500,
330
- ),
331
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
332
- ),
333
- const=TorchSchedulerConfig(
334
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
335
- hyperparams=ConfigDict[typing.Any](
336
- factor=1.0,
337
- total_iters=300000,
338
- ),
339
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
340
- ),
341
- ),
342
- )
343
- INFO 2026-04-06 11:29:31,546 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
344
- INFO 2026-04-06 11:29:34,908 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
345
- INFO 2026-04-06 11:29:43,404 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
346
- INFO 2026-04-06 11:29:47,554 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
347
- INFO 2026-04-06 11:29:51,073 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
348
- INFO 2026-04-06 11:29:51,073 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=16165.0, dataset_name=bridge_paraphrase_k5
349
- INFO 2026-04-06 11:29:51,112 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183239, rank=5
350
- INFO 2026-04-06 11:29:51,908 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=5, status=start
351
- INFO 2026-04-06 11:29:51,947 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
352
- INFO 2026-04-06 11:29:51,949 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=20403.0
353
- INFO 2026-04-06 11:29:51,956 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=5, status=start
354
- INFO 2026-04-06 11:31:39,838 barrel.train.core.trainer.trainer Restore session checkpoint rank=5, status=success
355
- INFO 2026-04-06 11:32:01,842 barrel.train.core.trainer.trainer session_init status=success, rank=5
356
- INFO 2026-04-06 11:32:01,844 barrel.train.core.trainer.trainer start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
357
- INFO 2026-04-06 11:32:02,002 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-7, worker_pid=3492775, parent_pid=3489445, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
358
- INFO 2026-04-06 11:32:02,028 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-7, worker_pid=3492785, parent_pid=3489445, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
359
- INFO 2026-04-06 11:35:29,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
360
- INFO 2026-04-06 11:38:35,202 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
361
- INFO 2026-04-06 11:41:41,427 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
362
- INFO 2026-04-06 11:44:46,849 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
363
- INFO 2026-04-06 11:47:52,317 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
364
- INFO 2026-04-06 11:50:56,956 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
365
- INFO 2026-04-06 11:54:03,028 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
366
- INFO 2026-04-06 11:57:07,422 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
367
- INFO 2026-04-06 12:00:13,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
368
- INFO 2026-04-06 12:03:19,351 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
369
- INFO 2026-04-06 12:06:23,696 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
370
- INFO 2026-04-06 12:09:29,448 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
371
- INFO 2026-04-06 12:12:35,570 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
372
- INFO 2026-04-06 12:15:41,418 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
373
- INFO 2026-04-06 12:18:46,706 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
374
- INFO 2026-04-06 12:21:51,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
375
- INFO 2026-04-06 12:24:57,745 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
376
- INFO 2026-04-06 12:28:01,897 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
377
- INFO 2026-04-06 12:31:07,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
378
- INFO 2026-04-06 12:34:11,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
379
- INFO 2026-04-06 12:37:16,739 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
380
- INFO 2026-04-06 12:40:21,217 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
381
- INFO 2026-04-06 12:43:25,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
382
- INFO 2026-04-06 12:46:31,794 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
383
- INFO 2026-04-06 12:49:37,506 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
384
- INFO 2026-04-06 12:52:43,062 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
385
- INFO 2026-04-06 12:55:47,772 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
386
- INFO 2026-04-06 12:58:53,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
387
- INFO 2026-04-06 13:01:58,050 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
388
- INFO 2026-04-06 13:05:03,714 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
389
- INFO 2026-04-06 13:08:09,158 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
390
- INFO 2026-04-06 13:11:13,197 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
391
- INFO 2026-04-06 13:14:19,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
392
- INFO 2026-04-06 13:17:25,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
393
- INFO 2026-04-06 13:20:30,201 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
394
- INFO 2026-04-06 13:23:35,857 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
395
- INFO 2026-04-06 13:26:41,883 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
396
- INFO 2026-04-06 13:29:48,084 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
397
- INFO 2026-04-06 13:32:52,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
398
- INFO 2026-04-06 13:35:57,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
399
- INFO 2026-04-06 13:39:03,081 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
400
- INFO 2026-04-06 13:42:10,403 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
401
- INFO 2026-04-06 13:45:14,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
402
- INFO 2026-04-06 13:48:20,973 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
403
- INFO 2026-04-06 13:51:25,864 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
404
- INFO 2026-04-06 13:54:30,720 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
405
- INFO 2026-04-06 13:57:36,003 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
406
- INFO 2026-04-06 14:00:41,877 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
407
- INFO 2026-04-06 14:03:49,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
408
- INFO 2026-04-06 14:07:04,771 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
409
- INFO 2026-04-06 14:10:39,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
410
- INFO 2026-04-06 14:13:44,081 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
411
- INFO 2026-04-06 14:16:49,284 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
412
- INFO 2026-04-06 14:19:53,530 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
413
- INFO 2026-04-06 14:22:58,783 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
414
- INFO 2026-04-06 14:26:03,761 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
415
- INFO 2026-04-06 14:29:07,809 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
416
- INFO 2026-04-06 14:32:11,852 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
417
- INFO 2026-04-06 14:35:16,304 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
418
- INFO 2026-04-06 14:38:22,035 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
419
- INFO 2026-04-06 14:41:25,476 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
420
- INFO 2026-04-06 14:44:30,532 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
421
- INFO 2026-04-06 14:47:36,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
422
- INFO 2026-04-06 14:50:40,768 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
423
- INFO 2026-04-06 14:53:45,052 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
424
- INFO 2026-04-06 14:56:50,253 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
425
- INFO 2026-04-06 14:59:56,687 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
426
- INFO 2026-04-06 15:03:02,324 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
427
- INFO 2026-04-07 00:41:05,434 barrel.train.components.optimizers.optimizer_module init_optimizer rank=5, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
428
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
429
- hyperparams=ConfigDict[typing.Any](
430
- lr=5e-05,
431
- weight_decay=0.0,
432
- ),
433
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
434
- torch_optimizer='torch.optim.AdamW',
435
- ), scheduler=SequentialSchedulerConfig(
436
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
437
- milestones=ConfigList[int](
438
- 500,
439
- ),
440
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
441
- warmup=TorchSchedulerConfig(
442
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
443
- hyperparams=ConfigDict[typing.Any](
444
- start_factor=0.0002,
445
- end_factor=1.0,
446
- total_iters=500,
447
- ),
448
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
449
- ),
450
- const=TorchSchedulerConfig(
451
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
452
- hyperparams=ConfigDict[typing.Any](
453
- factor=1.0,
454
- total_iters=300000,
455
- ),
456
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
457
- ),
458
- ),
459
- )
460
- INFO 2026-04-07 00:41:05,435 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
461
- INFO 2026-04-07 00:41:08,948 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
462
- INFO 2026-04-07 00:41:26,272 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
463
- INFO 2026-04-07 00:41:36,471 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
464
- INFO 2026-04-07 00:41:40,790 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
465
- INFO 2026-04-07 00:41:40,791 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=31842.4, dataset_name=bridge_paraphrase_k5
466
- INFO 2026-04-07 00:41:40,833 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183239, rank=5
467
- INFO 2026-04-07 00:41:41,660 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=5, status=start
468
- INFO 2026-04-07 00:41:41,700 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
469
- INFO 2026-04-07 00:41:41,701 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=36265.700000000004
470
- INFO 2026-04-07 00:41:41,706 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=5, status=start
471
- INFO 2026-04-07 00:43:48,408 barrel.train.core.trainer.trainer Restore session checkpoint rank=5, status=success
472
- INFO 2026-04-07 00:44:10,422 barrel.train.core.trainer.trainer session_init status=success, rank=5
473
- INFO 2026-04-07 00:44:10,423 barrel.train.core.trainer.trainer start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
474
- INFO 2026-04-07 00:44:10,495 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=3313513, parent_pid=3310183, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
475
- INFO 2026-04-07 00:44:10,551 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=3313548, parent_pid=3310183, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
476
- INFO 2026-04-07 00:47:36,030 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
477
- INFO 2026-04-07 00:50:38,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
478
- INFO 2026-04-07 00:53:42,314 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
479
- INFO 2026-04-07 00:56:45,084 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
480
- INFO 2026-04-07 00:59:47,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
481
- INFO 2026-04-07 01:02:50,206 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
482
- INFO 2026-04-07 01:05:53,527 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
483
- INFO 2026-04-07 01:08:56,071 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
484
- INFO 2026-04-07 01:11:58,572 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
485
- INFO 2026-04-07 01:15:02,202 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
486
- INFO 2026-04-07 01:18:05,101 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
487
- INFO 2026-04-07 01:21:08,472 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
488
- INFO 2026-04-07 01:24:11,700 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
489
- INFO 2026-04-07 01:27:15,143 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
490
- INFO 2026-04-07 01:30:16,886 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
491
- INFO 2026-04-07 01:33:19,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
492
- INFO 2026-04-07 01:36:22,454 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
493
- INFO 2026-04-07 01:39:25,144 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
494
- INFO 2026-04-07 01:42:29,625 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
495
- INFO 2026-04-07 01:45:32,634 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
496
- INFO 2026-04-07 01:48:36,836 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
497
- INFO 2026-04-07 01:51:39,520 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
498
- INFO 2026-04-07 01:54:42,912 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
499
- INFO 2026-04-07 01:57:46,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
500
- INFO 2026-04-07 02:00:50,170 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
501
- INFO 2026-04-07 02:03:54,280 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
502
- INFO 2026-04-07 02:06:57,975 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
503
- INFO 2026-04-07 02:10:02,505 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
504
- INFO 2026-04-07 02:13:05,977 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
505
- INFO 2026-04-07 02:16:10,140 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
506
- INFO 2026-04-07 02:19:14,248 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
507
- INFO 2026-04-07 02:22:17,141 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
508
- INFO 2026-04-07 02:25:21,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
509
- INFO 2026-04-07 02:28:24,737 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
510
- INFO 2026-04-07 02:31:29,707 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
511
- INFO 2026-04-07 02:34:33,437 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
512
- INFO 2026-04-07 02:37:37,742 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
513
- INFO 2026-04-07 02:40:41,391 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
514
- INFO 2026-04-07 02:43:44,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
515
- INFO 2026-04-07 02:46:48,387 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
516
- INFO 2026-04-07 02:49:52,443 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
517
- INFO 2026-04-07 02:52:56,836 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
518
- INFO 2026-04-07 02:56:01,912 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
519
- INFO 2026-04-07 02:59:07,348 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
520
- INFO 2026-04-07 03:02:11,728 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
521
- INFO 2026-04-07 03:05:16,730 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
522
- INFO 2026-04-07 03:08:20,102 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
523
- INFO 2026-04-07 03:11:23,980 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
524
- INFO 2026-04-07 03:14:28,690 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
525
- INFO 2026-04-07 03:17:51,392 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
526
- INFO 2026-04-07 03:18:34,222 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=5
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c185088a0665e7c6ae315b56822047d42ed5c9450b199d13cb4ebed348ff088
3
+ size 64010
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_6.log CHANGED
@@ -1,526 +1,3 @@
1
- INFO 2026-04-05 07:47:39,857 barrel.train.components.optimizers.optimizer_module init_optimizer rank=6, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
2
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
3
- hyperparams=ConfigDict[typing.Any](
4
- lr=5e-05,
5
- weight_decay=0.0,
6
- ),
7
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
8
- torch_optimizer='torch.optim.AdamW',
9
- ), scheduler=SequentialSchedulerConfig(
10
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
11
- milestones=ConfigList[int](
12
- 500,
13
- ),
14
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
15
- warmup=TorchSchedulerConfig(
16
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
17
- hyperparams=ConfigDict[typing.Any](
18
- start_factor=0.0002,
19
- end_factor=1.0,
20
- total_iters=500,
21
- ),
22
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
23
- ),
24
- const=TorchSchedulerConfig(
25
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
26
- hyperparams=ConfigDict[typing.Any](
27
- factor=1.0,
28
- total_iters=300000,
29
- ),
30
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
31
- ),
32
- ),
33
- )
34
- INFO 2026-04-05 07:47:39,858 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
35
- INFO 2026-04-05 07:47:42,026 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge
36
- INFO 2026-04-05 07:48:07,528 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
37
- INFO 2026-04-05 07:48:11,830 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
38
- INFO 2026-04-05 07:48:11,830 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=29804.1, dataset_name=bridge
39
- INFO 2026-04-05 07:48:11,870 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183295, rank=6
40
- INFO 2026-04-05 07:48:12,722 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge rank=6, status=start
41
- INFO 2026-04-05 07:48:12,763 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
42
- INFO 2026-04-05 07:48:12,764 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=32905.700000000004
43
- INFO 2026-04-05 07:48:34,459 barrel.train.core.trainer.trainer session_init status=success, rank=6
44
- INFO 2026-04-05 07:48:34,460 barrel.train.core.trainer.trainer start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
45
- INFO 2026-04-05 07:48:34,519 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-2, worker_pid=3875750, parent_pid=3872895, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
46
- INFO 2026-04-05 07:48:34,546 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-2, worker_pid=3875764, parent_pid=3872895, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
47
- INFO 2026-04-05 07:51:52,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
48
- INFO 2026-04-05 07:54:44,997 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
49
- INFO 2026-04-05 07:57:35,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
50
- INFO 2026-04-05 08:00:26,862 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
51
- INFO 2026-04-05 08:03:18,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
52
- INFO 2026-04-05 08:06:09,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
53
- INFO 2026-04-05 08:09:01,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
54
- INFO 2026-04-05 08:11:53,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
55
- INFO 2026-04-05 08:14:44,578 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
56
- INFO 2026-04-05 08:17:37,375 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
57
- INFO 2026-04-05 08:20:30,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
58
- INFO 2026-04-05 08:23:23,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
59
- INFO 2026-04-05 08:26:15,038 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
60
- INFO 2026-04-05 08:29:07,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
61
- INFO 2026-04-05 08:31:59,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
62
- INFO 2026-04-05 08:34:50,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
63
- INFO 2026-04-05 08:37:43,456 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
64
- INFO 2026-04-05 08:40:35,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
65
- INFO 2026-04-05 08:43:28,467 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
66
- INFO 2026-04-05 08:46:21,132 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
67
- INFO 2026-04-05 08:49:13,209 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
68
- INFO 2026-04-05 08:52:04,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
69
- INFO 2026-04-05 08:54:56,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
70
- INFO 2026-04-05 08:57:48,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
71
- INFO 2026-04-05 09:00:40,238 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
72
- INFO 2026-04-05 09:03:32,325 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
73
- INFO 2026-04-05 09:06:23,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
74
- INFO 2026-04-05 09:09:15,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
75
- INFO 2026-04-05 09:12:08,117 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
76
- INFO 2026-04-05 09:14:59,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
77
- INFO 2026-04-05 09:17:51,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
78
- INFO 2026-04-05 09:20:42,298 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
79
- INFO 2026-04-05 09:23:34,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
80
- INFO 2026-04-05 09:26:26,064 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
81
- INFO 2026-04-05 09:29:18,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
82
- INFO 2026-04-05 09:32:09,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
83
- INFO 2026-04-05 09:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
84
- INFO 2026-04-05 09:37:52,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
85
- INFO 2026-04-05 09:40:43,428 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
86
- INFO 2026-04-05 09:43:34,586 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
87
- INFO 2026-04-05 09:46:26,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
88
- INFO 2026-04-05 09:49:17,267 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
89
- INFO 2026-04-05 09:52:06,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
90
- INFO 2026-04-05 09:54:57,889 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
91
- INFO 2026-04-05 09:57:48,894 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
92
- INFO 2026-04-05 10:00:41,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
93
- INFO 2026-04-05 10:03:33,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
94
- INFO 2026-04-05 10:06:24,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
95
- INFO 2026-04-05 10:09:17,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
96
- INFO 2026-04-05 10:12:18,802 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
97
- INFO 2026-04-05 10:15:33,483 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
98
- INFO 2026-04-05 10:18:24,744 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
99
- INFO 2026-04-05 10:21:17,302 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
100
- INFO 2026-04-05 10:24:08,917 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
101
- INFO 2026-04-05 10:27:00,005 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
102
- INFO 2026-04-05 10:29:51,959 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
103
- INFO 2026-04-05 10:32:42,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
104
- INFO 2026-04-05 10:35:34,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
105
- INFO 2026-04-05 10:38:25,583 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
106
- INFO 2026-04-05 10:41:18,350 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
107
- INFO 2026-04-05 10:44:08,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
108
- INFO 2026-04-05 10:47:00,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
109
- INFO 2026-04-05 10:49:51,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
110
- INFO 2026-04-05 10:52:43,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
111
- INFO 2026-04-05 10:55:35,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
112
- INFO 2026-04-05 10:58:26,949 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
113
- INFO 2026-04-05 11:01:18,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
114
- INFO 2026-04-05 11:04:10,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
115
- INFO 2026-04-05 11:07:02,461 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
116
- INFO 2026-04-05 11:09:53,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
117
- INFO 2026-04-05 11:12:45,451 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
118
- INFO 2026-04-05 11:15:36,492 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
119
- INFO 2026-04-05 11:18:28,400 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
120
- INFO 2026-04-05 11:21:20,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
121
- INFO 2026-04-05 11:24:11,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
122
- INFO 2026-04-05 11:27:02,198 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
123
- INFO 2026-04-05 11:29:53,736 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
124
- INFO 2026-04-05 11:32:44,011 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
125
- INFO 2026-04-05 11:35:35,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
126
- INFO 2026-04-05 11:38:25,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
127
- INFO 2026-04-05 11:41:15,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
128
- INFO 2026-04-05 11:44:05,134 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
129
- INFO 2026-04-05 11:46:56,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
130
- INFO 2026-04-05 11:49:47,469 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
131
- INFO 2026-04-05 11:52:38,563 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
132
- INFO 2026-04-05 11:55:30,290 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
133
- INFO 2026-04-05 11:58:21,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
134
- INFO 2026-04-05 12:01:13,424 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
135
- INFO 2026-04-05 12:04:04,015 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
136
- INFO 2026-04-05 12:06:55,195 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
137
- INFO 2026-04-05 12:09:46,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
138
- INFO 2026-04-05 12:12:37,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
139
- INFO 2026-04-05 12:15:28,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
140
- INFO 2026-04-05 12:18:19,903 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
141
- INFO 2026-04-05 12:21:11,565 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
142
- INFO 2026-04-05 12:24:02,399 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
143
- INFO 2026-04-05 12:26:54,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
144
- INFO 2026-04-05 12:29:45,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
145
- INFO 2026-04-05 12:32:36,286 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
146
- INFO 2026-04-05 12:35:36,928 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
147
- INFO 2026-04-05 12:38:50,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
148
- INFO 2026-04-05 12:41:41,343 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
149
- INFO 2026-04-05 12:44:32,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
150
- INFO 2026-04-05 12:47:23,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
151
- INFO 2026-04-05 12:50:13,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
152
- INFO 2026-04-05 12:53:06,274 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
153
- INFO 2026-04-05 12:55:56,979 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
154
- INFO 2026-04-05 12:58:48,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
155
- INFO 2026-04-05 13:01:39,887 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
156
- INFO 2026-04-05 13:04:31,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
157
- INFO 2026-04-05 13:07:22,319 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
158
- INFO 2026-04-05 13:10:15,398 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
159
- INFO 2026-04-05 13:13:06,519 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
160
- INFO 2026-04-05 13:15:57,164 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
161
- INFO 2026-04-05 13:18:49,647 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
162
- INFO 2026-04-05 13:21:40,162 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
163
- INFO 2026-04-05 13:24:31,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
164
- INFO 2026-04-05 13:27:22,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
165
- INFO 2026-04-05 13:30:13,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
166
- INFO 2026-04-05 13:33:06,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
167
- INFO 2026-04-05 13:35:57,606 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
168
- INFO 2026-04-05 13:38:47,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
169
- INFO 2026-04-05 13:41:40,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
170
- INFO 2026-04-05 13:44:31,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
171
- INFO 2026-04-05 13:47:23,254 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
172
- INFO 2026-04-05 13:50:14,660 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
173
- INFO 2026-04-05 13:53:06,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
174
- INFO 2026-04-05 13:55:58,107 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
175
- INFO 2026-04-05 13:58:50,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
176
- INFO 2026-04-05 14:01:41,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
177
- INFO 2026-04-05 14:04:32,278 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
178
- INFO 2026-04-05 14:07:24,300 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
179
- INFO 2026-04-05 14:10:15,786 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
180
- INFO 2026-04-05 14:13:07,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
181
- INFO 2026-04-05 14:15:58,762 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
182
- INFO 2026-04-05 14:18:49,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
183
- INFO 2026-04-05 14:21:40,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
184
- INFO 2026-04-05 14:24:31,803 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
185
- INFO 2026-04-05 14:27:22,607 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
186
- INFO 2026-04-05 14:30:14,380 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
187
- INFO 2026-04-05 14:33:06,049 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
188
- INFO 2026-04-05 14:35:56,094 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
189
- INFO 2026-04-05 14:38:48,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
190
- INFO 2026-04-05 14:41:40,352 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
191
- INFO 2026-04-05 14:44:32,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
192
- INFO 2026-04-05 14:47:25,215 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
193
- INFO 2026-04-05 14:50:15,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
194
- INFO 2026-04-05 14:53:07,347 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
195
- INFO 2026-04-05 14:56:00,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
196
- INFO 2026-04-05 14:59:02,280 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
197
- INFO 2026-04-05 14:59:35,377 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=6
198
- INFO 2026-04-06 07:33:53,201 barrel.train.components.optimizers.optimizer_module init_optimizer rank=6, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
199
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
200
- hyperparams=ConfigDict[typing.Any](
201
- lr=5e-05,
202
- weight_decay=0.0,
203
- ),
204
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
205
- torch_optimizer='torch.optim.AdamW',
206
- ), scheduler=SequentialSchedulerConfig(
207
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
208
- milestones=ConfigList[int](
209
- 500,
210
- ),
211
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
212
- warmup=TorchSchedulerConfig(
213
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
214
- hyperparams=ConfigDict[typing.Any](
215
- start_factor=0.0002,
216
- end_factor=1.0,
217
- total_iters=500,
218
- ),
219
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
220
- ),
221
- const=TorchSchedulerConfig(
222
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
223
- hyperparams=ConfigDict[typing.Any](
224
- factor=1.0,
225
- total_iters=300000,
226
- ),
227
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
228
- ),
229
- ),
230
- )
231
- INFO 2026-04-06 07:33:53,203 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
232
- INFO 2026-04-06 07:33:56,239 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
233
- INFO 2026-04-06 07:34:13,865 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
234
- INFO 2026-04-06 07:34:24,372 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
235
- INFO 2026-04-06 07:34:28,401 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
236
- INFO 2026-04-06 07:34:28,401 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=32162.199999999997, dataset_name=bridge_paraphrase_k5
237
- INFO 2026-04-06 07:34:28,444 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183295, rank=6
238
- INFO 2026-04-06 07:34:29,334 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=6, status=start
239
- INFO 2026-04-06 07:34:29,376 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
240
- INFO 2026-04-06 07:34:29,378 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=36174.799999999996
241
- INFO 2026-04-06 07:34:29,385 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=6, status=start
242
- INFO 2026-04-06 07:36:05,852 barrel.train.core.trainer.trainer Restore session checkpoint rank=6, status=success
243
- INFO 2026-04-06 07:36:27,877 barrel.train.core.trainer.trainer session_init status=success, rank=6
244
- INFO 2026-04-06 07:36:27,879 barrel.train.core.trainer.trainer start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
245
- INFO 2026-04-06 07:36:27,950 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=2465175, parent_pid=2461891, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
246
- INFO 2026-04-06 07:36:27,996 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=2465219, parent_pid=2461891, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
247
- INFO 2026-04-06 07:39:43,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
248
- INFO 2026-04-06 07:42:35,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
249
- INFO 2026-04-06 07:45:26,710 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
250
- INFO 2026-04-06 07:48:18,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
251
- INFO 2026-04-06 07:51:08,588 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
252
- INFO 2026-04-06 07:54:00,463 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
253
- INFO 2026-04-06 07:56:51,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
254
- INFO 2026-04-06 07:59:43,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
255
- INFO 2026-04-06 08:02:34,655 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
256
- INFO 2026-04-06 08:05:26,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
257
- INFO 2026-04-06 08:08:18,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
258
- INFO 2026-04-06 08:11:11,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
259
- INFO 2026-04-06 08:14:03,148 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
260
- INFO 2026-04-06 08:16:57,119 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
261
- INFO 2026-04-06 08:19:48,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
262
- INFO 2026-04-06 08:22:39,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
263
- INFO 2026-04-06 08:25:31,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
264
- INFO 2026-04-06 08:28:25,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
265
- INFO 2026-04-06 08:31:16,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
266
- INFO 2026-04-06 08:34:09,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
267
- INFO 2026-04-06 08:37:01,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
268
- INFO 2026-04-06 08:39:53,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
269
- INFO 2026-04-06 08:42:45,381 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
270
- INFO 2026-04-06 08:45:38,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
271
- INFO 2026-04-06 08:48:30,988 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
272
- INFO 2026-04-06 08:51:23,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
273
- INFO 2026-04-06 08:54:13,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
274
- INFO 2026-04-06 08:57:05,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
275
- INFO 2026-04-06 08:59:57,948 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
276
- INFO 2026-04-06 09:02:49,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
277
- INFO 2026-04-06 09:05:41,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
278
- INFO 2026-04-06 09:08:33,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
279
- INFO 2026-04-06 09:11:25,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
280
- INFO 2026-04-06 09:14:17,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
281
- INFO 2026-04-06 09:17:08,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
282
- INFO 2026-04-06 09:20:01,792 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
283
- INFO 2026-04-06 09:22:53,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
284
- INFO 2026-04-06 09:25:46,358 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
285
- INFO 2026-04-06 09:28:40,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
286
- INFO 2026-04-06 09:31:33,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
287
- INFO 2026-04-06 09:34:24,731 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
288
- INFO 2026-04-06 09:37:17,501 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
289
- INFO 2026-04-06 09:40:08,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
290
- INFO 2026-04-06 09:43:00,698 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
291
- INFO 2026-04-06 09:45:52,659 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
292
- INFO 2026-04-06 09:48:44,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
293
- INFO 2026-04-06 09:51:36,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
294
- INFO 2026-04-06 09:54:29,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
295
- INFO 2026-04-06 09:57:20,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
296
- INFO 2026-04-06 10:00:22,907 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
297
- INFO 2026-04-06 10:03:48,884 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
298
- INFO 2026-04-06 10:06:52,105 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
299
- INFO 2026-04-06 10:09:59,029 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
300
- INFO 2026-04-06 10:13:05,841 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
301
- INFO 2026-04-06 10:16:11,935 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
302
- INFO 2026-04-06 10:19:15,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
303
- INFO 2026-04-06 10:22:24,669 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
304
- INFO 2026-04-06 10:25:33,552 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
305
- INFO 2026-04-06 10:28:42,181 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
306
- INFO 2026-04-06 10:31:51,520 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
307
- INFO 2026-04-06 10:35:00,447 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
308
- INFO 2026-04-06 10:38:08,612 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
309
- INFO 2026-04-06 10:41:15,603 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
310
- INFO 2026-04-06 11:29:31,542 barrel.train.components.optimizers.optimizer_module init_optimizer rank=6, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
311
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
312
- hyperparams=ConfigDict[typing.Any](
313
- lr=5e-05,
314
- weight_decay=0.0,
315
- ),
316
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
317
- torch_optimizer='torch.optim.AdamW',
318
- ), scheduler=SequentialSchedulerConfig(
319
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
320
- milestones=ConfigList[int](
321
- 500,
322
- ),
323
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
324
- warmup=TorchSchedulerConfig(
325
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
326
- hyperparams=ConfigDict[typing.Any](
327
- start_factor=0.0002,
328
- end_factor=1.0,
329
- total_iters=500,
330
- ),
331
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
332
- ),
333
- const=TorchSchedulerConfig(
334
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
335
- hyperparams=ConfigDict[typing.Any](
336
- factor=1.0,
337
- total_iters=300000,
338
- ),
339
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
340
- ),
341
- ),
342
- )
343
- INFO 2026-04-06 11:29:31,545 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
344
- INFO 2026-04-06 11:29:34,960 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
345
- INFO 2026-04-06 11:29:43,764 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
346
- INFO 2026-04-06 11:29:48,060 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
347
- INFO 2026-04-06 11:29:51,777 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
348
- INFO 2026-04-06 11:29:51,777 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=16817.3, dataset_name=bridge_paraphrase_k5
349
- INFO 2026-04-06 11:29:51,818 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183295, rank=6
350
- INFO 2026-04-06 11:29:52,637 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=6, status=start
351
- INFO 2026-04-06 11:29:52,678 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
352
- INFO 2026-04-06 11:29:52,679 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=21134.0
353
- INFO 2026-04-06 11:29:52,684 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=6, status=start
354
- INFO 2026-04-06 11:31:39,839 barrel.train.core.trainer.trainer Restore session checkpoint rank=6, status=success
355
- INFO 2026-04-06 11:32:01,842 barrel.train.core.trainer.trainer session_init status=success, rank=6
356
- INFO 2026-04-06 11:32:01,844 barrel.train.core.trainer.trainer start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
357
- INFO 2026-04-06 11:32:02,000 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-7, worker_pid=3492774, parent_pid=3489446, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
358
- INFO 2026-04-06 11:32:02,023 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-7, worker_pid=3492782, parent_pid=3489446, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
359
- INFO 2026-04-06 11:35:29,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
360
- INFO 2026-04-06 11:38:35,206 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
361
- INFO 2026-04-06 11:41:41,429 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
362
- INFO 2026-04-06 11:44:46,850 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
363
- INFO 2026-04-06 11:47:52,316 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
364
- INFO 2026-04-06 11:50:56,956 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
365
- INFO 2026-04-06 11:54:03,028 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
366
- INFO 2026-04-06 11:57:07,422 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
367
- INFO 2026-04-06 12:00:13,667 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
368
- INFO 2026-04-06 12:03:19,351 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
369
- INFO 2026-04-06 12:06:23,697 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
370
- INFO 2026-04-06 12:09:29,450 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
371
- INFO 2026-04-06 12:12:35,575 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
372
- INFO 2026-04-06 12:15:41,418 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
373
- INFO 2026-04-06 12:18:46,706 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
374
- INFO 2026-04-06 12:21:51,864 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
375
- INFO 2026-04-06 12:24:57,746 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
376
- INFO 2026-04-06 12:28:01,899 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
377
- INFO 2026-04-06 12:31:07,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
378
- INFO 2026-04-06 12:34:11,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
379
- INFO 2026-04-06 12:37:16,741 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
380
- INFO 2026-04-06 12:40:21,219 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
381
- INFO 2026-04-06 12:43:25,435 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
382
- INFO 2026-04-06 12:46:31,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
383
- INFO 2026-04-06 12:49:37,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
384
- INFO 2026-04-06 12:52:43,063 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
385
- INFO 2026-04-06 12:55:47,774 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
386
- INFO 2026-04-06 12:58:53,488 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
387
- INFO 2026-04-06 13:01:58,055 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
388
- INFO 2026-04-06 13:05:03,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
389
- INFO 2026-04-06 13:08:09,159 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
390
- INFO 2026-04-06 13:11:13,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
391
- INFO 2026-04-06 13:14:19,369 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
392
- INFO 2026-04-06 13:17:25,230 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
393
- INFO 2026-04-06 13:20:30,201 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
394
- INFO 2026-04-06 13:23:35,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
395
- INFO 2026-04-06 13:26:41,884 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
396
- INFO 2026-04-06 13:29:48,087 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
397
- INFO 2026-04-06 13:32:52,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
398
- INFO 2026-04-06 13:35:57,627 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
399
- INFO 2026-04-06 13:39:03,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
400
- INFO 2026-04-06 13:42:10,406 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
401
- INFO 2026-04-06 13:45:14,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
402
- INFO 2026-04-06 13:48:20,970 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
403
- INFO 2026-04-06 13:51:25,882 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
404
- INFO 2026-04-06 13:54:30,723 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
405
- INFO 2026-04-06 13:57:36,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
406
- INFO 2026-04-06 14:00:41,877 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
407
- INFO 2026-04-06 14:03:49,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
408
- INFO 2026-04-06 14:07:04,670 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
409
- INFO 2026-04-06 14:10:39,272 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
410
- INFO 2026-04-06 14:13:44,086 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
411
- INFO 2026-04-06 14:16:49,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
412
- INFO 2026-04-06 14:19:53,531 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
413
- INFO 2026-04-06 14:22:58,790 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
414
- INFO 2026-04-06 14:26:03,764 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
415
- INFO 2026-04-06 14:29:07,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
416
- INFO 2026-04-06 14:32:11,856 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
417
- INFO 2026-04-06 14:35:16,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
418
- INFO 2026-04-06 14:38:22,037 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
419
- INFO 2026-04-06 14:41:25,477 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
420
- INFO 2026-04-06 14:44:30,530 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
421
- INFO 2026-04-06 14:47:36,056 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
422
- INFO 2026-04-06 14:50:40,770 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
423
- INFO 2026-04-06 14:53:45,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
424
- INFO 2026-04-06 14:56:50,256 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
425
- INFO 2026-04-06 14:59:56,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
426
- INFO 2026-04-06 15:03:02,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
427
- INFO 2026-04-07 00:41:05,408 barrel.train.components.optimizers.optimizer_module init_optimizer rank=6, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
428
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
429
- hyperparams=ConfigDict[typing.Any](
430
- lr=5e-05,
431
- weight_decay=0.0,
432
- ),
433
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
434
- torch_optimizer='torch.optim.AdamW',
435
- ), scheduler=SequentialSchedulerConfig(
436
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
437
- milestones=ConfigList[int](
438
- 500,
439
- ),
440
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
441
- warmup=TorchSchedulerConfig(
442
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
443
- hyperparams=ConfigDict[typing.Any](
444
- start_factor=0.0002,
445
- end_factor=1.0,
446
- total_iters=500,
447
- ),
448
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
449
- ),
450
- const=TorchSchedulerConfig(
451
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
452
- hyperparams=ConfigDict[typing.Any](
453
- factor=1.0,
454
- total_iters=300000,
455
- ),
456
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
457
- ),
458
- ),
459
- )
460
- INFO 2026-04-07 00:41:05,410 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
461
- INFO 2026-04-07 00:41:08,936 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
462
- INFO 2026-04-07 00:41:26,311 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
463
- INFO 2026-04-07 00:41:36,523 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
464
- INFO 2026-04-07 00:41:40,869 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
465
- INFO 2026-04-07 00:41:40,869 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=31933.6, dataset_name=bridge_paraphrase_k5
466
- INFO 2026-04-07 00:41:40,910 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183295, rank=6
467
- INFO 2026-04-07 00:41:41,765 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=6, status=start
468
- INFO 2026-04-07 00:41:41,805 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
469
- INFO 2026-04-07 00:41:41,806 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=36396.2
470
- INFO 2026-04-07 00:41:41,811 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=6, status=start
471
- INFO 2026-04-07 00:43:48,407 barrel.train.core.trainer.trainer Restore session checkpoint rank=6, status=success
472
- INFO 2026-04-07 00:44:10,422 barrel.train.core.trainer.trainer session_init status=success, rank=6
473
- INFO 2026-04-07 00:44:10,423 barrel.train.core.trainer.trainer start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
474
- INFO 2026-04-07 00:44:10,504 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=3313522, parent_pid=3310184, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
475
- INFO 2026-04-07 00:44:10,548 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=3313561, parent_pid=3310184, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
476
- INFO 2026-04-07 00:47:36,030 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
477
- INFO 2026-04-07 00:50:38,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
478
- INFO 2026-04-07 00:53:42,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
479
- INFO 2026-04-07 00:56:45,084 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
480
- INFO 2026-04-07 00:59:47,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
481
- INFO 2026-04-07 01:02:50,208 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
482
- INFO 2026-04-07 01:05:53,527 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
483
- INFO 2026-04-07 01:08:56,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
484
- INFO 2026-04-07 01:11:58,574 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
485
- INFO 2026-04-07 01:15:02,200 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
486
- INFO 2026-04-07 01:18:05,102 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
487
- INFO 2026-04-07 01:21:08,470 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
488
- INFO 2026-04-07 01:24:11,699 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
489
- INFO 2026-04-07 01:27:15,143 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
490
- INFO 2026-04-07 01:30:16,883 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
491
- INFO 2026-04-07 01:33:19,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
492
- INFO 2026-04-07 01:36:22,453 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
493
- INFO 2026-04-07 01:39:25,148 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
494
- INFO 2026-04-07 01:42:29,624 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
495
- INFO 2026-04-07 01:45:32,635 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
496
- INFO 2026-04-07 01:48:36,848 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
497
- INFO 2026-04-07 01:51:39,488 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
498
- INFO 2026-04-07 01:54:42,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
499
- INFO 2026-04-07 01:57:46,407 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
500
- INFO 2026-04-07 02:00:50,171 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
501
- INFO 2026-04-07 02:03:54,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
502
- INFO 2026-04-07 02:06:57,975 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
503
- INFO 2026-04-07 02:10:02,506 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
504
- INFO 2026-04-07 02:13:05,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
505
- INFO 2026-04-07 02:16:10,139 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
506
- INFO 2026-04-07 02:19:14,241 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
507
- INFO 2026-04-07 02:22:17,153 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
508
- INFO 2026-04-07 02:25:21,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
509
- INFO 2026-04-07 02:28:24,735 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
510
- INFO 2026-04-07 02:31:29,705 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
511
- INFO 2026-04-07 02:34:33,433 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
512
- INFO 2026-04-07 02:37:37,739 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
513
- INFO 2026-04-07 02:40:41,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
514
- INFO 2026-04-07 02:43:44,850 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
515
- INFO 2026-04-07 02:46:48,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
516
- INFO 2026-04-07 02:49:52,439 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
517
- INFO 2026-04-07 02:52:56,832 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
518
- INFO 2026-04-07 02:56:01,908 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
519
- INFO 2026-04-07 02:59:07,330 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
520
- INFO 2026-04-07 03:02:11,724 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
521
- INFO 2026-04-07 03:05:16,730 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
522
- INFO 2026-04-07 03:08:20,116 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
523
- INFO 2026-04-07 03:11:23,981 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
524
- INFO 2026-04-07 03:14:28,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
525
- INFO 2026-04-07 03:17:51,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
526
- INFO 2026-04-07 03:18:34,222 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=6
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:28436bacbb6ea3b7e483252210135f9d5c163b54976a0a20ff7e27b3f6b99a27
3
+ size 64032
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_7.log CHANGED
@@ -1,526 +1,3 @@
1
- INFO 2026-04-05 07:47:39,914 barrel.train.components.optimizers.optimizer_module init_optimizer rank=7, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
2
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
3
- hyperparams=ConfigDict[typing.Any](
4
- lr=5e-05,
5
- weight_decay=0.0,
6
- ),
7
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
8
- torch_optimizer='torch.optim.AdamW',
9
- ), scheduler=SequentialSchedulerConfig(
10
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
11
- milestones=ConfigList[int](
12
- 500,
13
- ),
14
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
15
- warmup=TorchSchedulerConfig(
16
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
17
- hyperparams=ConfigDict[typing.Any](
18
- start_factor=0.0002,
19
- end_factor=1.0,
20
- total_iters=500,
21
- ),
22
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
23
- ),
24
- const=TorchSchedulerConfig(
25
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
26
- hyperparams=ConfigDict[typing.Any](
27
- factor=1.0,
28
- total_iters=300000,
29
- ),
30
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
31
- ),
32
- ),
33
- )
34
- INFO 2026-04-05 07:47:39,915 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
35
- INFO 2026-04-05 07:47:42,038 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge
36
- INFO 2026-04-05 07:48:03,856 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
37
- INFO 2026-04-05 07:48:08,037 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
38
- INFO 2026-04-05 07:48:08,037 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=25998.7, dataset_name=bridge
39
- INFO 2026-04-05 07:48:08,077 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183242, rank=7
40
- INFO 2026-04-05 07:48:08,930 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge rank=7, status=start
41
- INFO 2026-04-05 07:48:08,969 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
42
- INFO 2026-04-05 07:48:08,970 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=29055.3
43
- INFO 2026-04-05 07:48:34,458 barrel.train.core.trainer.trainer session_init status=success, rank=7
44
- INFO 2026-04-05 07:48:34,460 barrel.train.core.trainer.trainer start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
45
- INFO 2026-04-05 07:48:34,515 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-2, worker_pid=3875748, parent_pid=3872896, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
46
- INFO 2026-04-05 07:48:34,543 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-2, worker_pid=3875758, parent_pid=3872896, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
47
- INFO 2026-04-05 07:51:52,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
48
- INFO 2026-04-05 07:54:44,997 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
49
- INFO 2026-04-05 07:57:35,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
50
- INFO 2026-04-05 08:00:26,862 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
51
- INFO 2026-04-05 08:03:18,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
52
- INFO 2026-04-05 08:06:09,619 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
53
- INFO 2026-04-05 08:09:01,077 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
54
- INFO 2026-04-05 08:11:53,412 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
55
- INFO 2026-04-05 08:14:44,578 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
56
- INFO 2026-04-05 08:17:37,375 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
57
- INFO 2026-04-05 08:20:30,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
58
- INFO 2026-04-05 08:23:23,222 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
59
- INFO 2026-04-05 08:26:15,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
60
- INFO 2026-04-05 08:29:07,502 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
61
- INFO 2026-04-05 08:31:59,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
62
- INFO 2026-04-05 08:34:50,984 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
63
- INFO 2026-04-05 08:37:43,456 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
64
- INFO 2026-04-05 08:40:35,332 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
65
- INFO 2026-04-05 08:43:28,467 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
66
- INFO 2026-04-05 08:46:21,132 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
67
- INFO 2026-04-05 08:49:13,210 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
68
- INFO 2026-04-05 08:52:04,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
69
- INFO 2026-04-05 08:54:56,866 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
70
- INFO 2026-04-05 08:57:48,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
71
- INFO 2026-04-05 09:00:40,238 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
72
- INFO 2026-04-05 09:03:32,325 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
73
- INFO 2026-04-05 09:06:23,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
74
- INFO 2026-04-05 09:09:15,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
75
- INFO 2026-04-05 09:12:08,117 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
76
- INFO 2026-04-05 09:14:59,227 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
77
- INFO 2026-04-05 09:17:51,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
78
- INFO 2026-04-05 09:20:42,298 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
79
- INFO 2026-04-05 09:23:34,083 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
80
- INFO 2026-04-05 09:26:26,064 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
81
- INFO 2026-04-05 09:29:18,322 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
82
- INFO 2026-04-05 09:32:09,487 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
83
- INFO 2026-04-05 09:35:00,441 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
84
- INFO 2026-04-05 09:37:52,631 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
85
- INFO 2026-04-05 09:40:43,428 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
86
- INFO 2026-04-05 09:43:34,586 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
87
- INFO 2026-04-05 09:46:26,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
88
- INFO 2026-04-05 09:49:17,267 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
89
- INFO 2026-04-05 09:52:06,854 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
90
- INFO 2026-04-05 09:54:57,889 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
91
- INFO 2026-04-05 09:57:48,894 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
92
- INFO 2026-04-05 10:00:41,039 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
93
- INFO 2026-04-05 10:03:33,426 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
94
- INFO 2026-04-05 10:06:24,800 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
95
- INFO 2026-04-05 10:09:17,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
96
- INFO 2026-04-05 10:12:18,804 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
97
- INFO 2026-04-05 10:15:33,483 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
98
- INFO 2026-04-05 10:18:24,744 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
99
- INFO 2026-04-05 10:21:17,302 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
100
- INFO 2026-04-05 10:24:08,917 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
101
- INFO 2026-04-05 10:27:00,005 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
102
- INFO 2026-04-05 10:29:51,959 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
103
- INFO 2026-04-05 10:32:42,383 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
104
- INFO 2026-04-05 10:35:34,045 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
105
- INFO 2026-04-05 10:38:25,583 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
106
- INFO 2026-04-05 10:41:18,350 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
107
- INFO 2026-04-05 10:44:08,990 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
108
- INFO 2026-04-05 10:47:00,701 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
109
- INFO 2026-04-05 10:49:51,287 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
110
- INFO 2026-04-05 10:52:43,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
111
- INFO 2026-04-05 10:55:35,867 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
112
- INFO 2026-04-05 10:58:26,949 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
113
- INFO 2026-04-05 11:01:18,366 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
114
- INFO 2026-04-05 11:04:10,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
115
- INFO 2026-04-05 11:07:02,461 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
116
- INFO 2026-04-05 11:09:53,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
117
- INFO 2026-04-05 11:12:45,451 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
118
- INFO 2026-04-05 11:15:36,492 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
119
- INFO 2026-04-05 11:18:28,400 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
120
- INFO 2026-04-05 11:21:20,006 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
121
- INFO 2026-04-05 11:24:11,858 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
122
- INFO 2026-04-05 11:27:02,198 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
123
- INFO 2026-04-05 11:29:53,736 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
124
- INFO 2026-04-05 11:32:44,011 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
125
- INFO 2026-04-05 11:35:35,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
126
- INFO 2026-04-05 11:38:25,486 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
127
- INFO 2026-04-05 11:41:15,315 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
128
- INFO 2026-04-05 11:44:05,134 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
129
- INFO 2026-04-05 11:46:56,617 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
130
- INFO 2026-04-05 11:49:47,469 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
131
- INFO 2026-04-05 11:52:38,563 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
132
- INFO 2026-04-05 11:55:30,289 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
133
- INFO 2026-04-05 11:58:21,027 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
134
- INFO 2026-04-05 12:01:13,424 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
135
- INFO 2026-04-05 12:04:04,015 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
136
- INFO 2026-04-05 12:06:55,196 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
137
- INFO 2026-04-05 12:09:46,718 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
138
- INFO 2026-04-05 12:12:37,434 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
139
- INFO 2026-04-05 12:15:28,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
140
- INFO 2026-04-05 12:18:19,903 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
141
- INFO 2026-04-05 12:21:11,565 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
142
- INFO 2026-04-05 12:24:02,399 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
143
- INFO 2026-04-05 12:26:54,834 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
144
- INFO 2026-04-05 12:29:45,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
145
- INFO 2026-04-05 12:32:36,286 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
146
- INFO 2026-04-05 12:35:36,951 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
147
- INFO 2026-04-05 12:38:50,026 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
148
- INFO 2026-04-05 12:41:41,343 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
149
- INFO 2026-04-05 12:44:32,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
150
- INFO 2026-04-05 12:47:23,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
151
- INFO 2026-04-05 12:50:13,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
152
- INFO 2026-04-05 12:53:06,274 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
153
- INFO 2026-04-05 12:55:56,980 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
154
- INFO 2026-04-05 12:58:48,716 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
155
- INFO 2026-04-05 13:01:39,887 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
156
- INFO 2026-04-05 13:04:31,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
157
- INFO 2026-04-05 13:07:22,319 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
158
- INFO 2026-04-05 13:10:15,398 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
159
- INFO 2026-04-05 13:13:06,519 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
160
- INFO 2026-04-05 13:15:57,164 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
161
- INFO 2026-04-05 13:18:49,647 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
162
- INFO 2026-04-05 13:21:40,162 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
163
- INFO 2026-04-05 13:24:31,281 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
164
- INFO 2026-04-05 13:27:22,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
165
- INFO 2026-04-05 13:30:13,985 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
166
- INFO 2026-04-05 13:33:06,043 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
167
- INFO 2026-04-05 13:35:57,606 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
168
- INFO 2026-04-05 13:38:47,618 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
169
- INFO 2026-04-05 13:41:40,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
170
- INFO 2026-04-05 13:44:31,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
171
- INFO 2026-04-05 13:47:23,254 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
172
- INFO 2026-04-05 13:50:14,660 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
173
- INFO 2026-04-05 13:53:06,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
174
- INFO 2026-04-05 13:55:58,107 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
175
- INFO 2026-04-05 13:58:50,688 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
176
- INFO 2026-04-05 14:01:41,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
177
- INFO 2026-04-05 14:04:32,278 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
178
- INFO 2026-04-05 14:07:24,300 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
179
- INFO 2026-04-05 14:10:15,785 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
180
- INFO 2026-04-05 14:13:07,365 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
181
- INFO 2026-04-05 14:15:58,762 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
182
- INFO 2026-04-05 14:18:49,976 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
183
- INFO 2026-04-05 14:21:40,911 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
184
- INFO 2026-04-05 14:24:31,803 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
185
- INFO 2026-04-05 14:27:22,607 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
186
- INFO 2026-04-05 14:30:14,379 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
187
- INFO 2026-04-05 14:33:06,049 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
188
- INFO 2026-04-05 14:35:56,094 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
189
- INFO 2026-04-05 14:38:48,561 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
190
- INFO 2026-04-05 14:41:40,352 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
191
- INFO 2026-04-05 14:44:32,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
192
- INFO 2026-04-05 14:47:25,215 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
193
- INFO 2026-04-05 14:50:15,915 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
194
- INFO 2026-04-05 14:53:07,347 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
195
- INFO 2026-04-05 14:56:00,277 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
196
- INFO 2026-04-05 14:59:02,313 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
197
- INFO 2026-04-05 14:59:35,376 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=7
198
- INFO 2026-04-06 07:33:53,243 barrel.train.components.optimizers.optimizer_module init_optimizer rank=7, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
199
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
200
- hyperparams=ConfigDict[typing.Any](
201
- lr=5e-05,
202
- weight_decay=0.0,
203
- ),
204
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
205
- torch_optimizer='torch.optim.AdamW',
206
- ), scheduler=SequentialSchedulerConfig(
207
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
208
- milestones=ConfigList[int](
209
- 500,
210
- ),
211
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
212
- warmup=TorchSchedulerConfig(
213
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
214
- hyperparams=ConfigDict[typing.Any](
215
- start_factor=0.0002,
216
- end_factor=1.0,
217
- total_iters=500,
218
- ),
219
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
220
- ),
221
- const=TorchSchedulerConfig(
222
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
223
- hyperparams=ConfigDict[typing.Any](
224
- factor=1.0,
225
- total_iters=300000,
226
- ),
227
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
228
- ),
229
- ),
230
- )
231
- INFO 2026-04-06 07:33:53,246 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
232
- INFO 2026-04-06 07:33:56,074 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
233
- INFO 2026-04-06 07:34:10,300 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
234
- INFO 2026-04-06 07:34:17,978 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
235
- INFO 2026-04-06 07:34:22,270 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
236
- INFO 2026-04-06 07:34:22,270 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=26196.8, dataset_name=bridge_paraphrase_k5
237
- INFO 2026-04-06 07:34:22,313 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183242, rank=7
238
- INFO 2026-04-06 07:34:23,188 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=7, status=start
239
- INFO 2026-04-06 07:34:23,228 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
240
- INFO 2026-04-06 07:34:23,230 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=29983.399999999998
241
- INFO 2026-04-06 07:34:23,236 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=7, status=start
242
- INFO 2026-04-06 07:36:05,844 barrel.train.core.trainer.trainer Restore session checkpoint rank=7, status=success
243
- INFO 2026-04-06 07:36:27,876 barrel.train.core.trainer.trainer session_init status=success, rank=7
244
- INFO 2026-04-06 07:36:27,879 barrel.train.core.trainer.trainer start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
245
- INFO 2026-04-06 07:36:27,946 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=2465171, parent_pid=2461892, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
246
- INFO 2026-04-06 07:36:27,979 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=2465202, parent_pid=2461892, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
247
- INFO 2026-04-06 07:39:43,068 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
248
- INFO 2026-04-06 07:42:35,329 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
249
- INFO 2026-04-06 07:45:26,709 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
250
- INFO 2026-04-06 07:48:18,405 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
251
- INFO 2026-04-06 07:51:08,588 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
252
- INFO 2026-04-06 07:54:00,463 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
253
- INFO 2026-04-06 07:56:51,070 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
254
- INFO 2026-04-06 07:59:43,648 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
255
- INFO 2026-04-06 08:02:34,655 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
256
- INFO 2026-04-06 08:05:26,546 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
257
- INFO 2026-04-06 08:08:18,653 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
258
- INFO 2026-04-06 08:11:11,326 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
259
- INFO 2026-04-06 08:14:03,148 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
260
- INFO 2026-04-06 08:16:57,119 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
261
- INFO 2026-04-06 08:19:48,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
262
- INFO 2026-04-06 08:22:39,880 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
263
- INFO 2026-04-06 08:25:31,510 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
264
- INFO 2026-04-06 08:28:25,199 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
265
- INFO 2026-04-06 08:31:16,666 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
266
- INFO 2026-04-06 08:34:09,668 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
267
- INFO 2026-04-06 08:37:01,040 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
268
- INFO 2026-04-06 08:39:53,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
269
- INFO 2026-04-06 08:42:45,381 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
270
- INFO 2026-04-06 08:45:38,330 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
271
- INFO 2026-04-06 08:48:30,988 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
272
- INFO 2026-04-06 08:51:23,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
273
- INFO 2026-04-06 08:54:13,610 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
274
- INFO 2026-04-06 08:57:05,895 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
275
- INFO 2026-04-06 08:59:57,948 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
276
- INFO 2026-04-06 09:02:49,534 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
277
- INFO 2026-04-06 09:05:41,813 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
278
- INFO 2026-04-06 09:08:33,644 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
279
- INFO 2026-04-06 09:11:25,220 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
280
- INFO 2026-04-06 09:14:17,223 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
281
- INFO 2026-04-06 09:17:08,793 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
282
- INFO 2026-04-06 09:20:01,791 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
283
- INFO 2026-04-06 09:22:53,269 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
284
- INFO 2026-04-06 09:25:46,358 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
285
- INFO 2026-04-06 09:28:40,722 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
286
- INFO 2026-04-06 09:31:33,388 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
287
- INFO 2026-04-06 09:34:24,732 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
288
- INFO 2026-04-06 09:37:17,501 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
289
- INFO 2026-04-06 09:40:08,460 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
290
- INFO 2026-04-06 09:43:00,698 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
291
- INFO 2026-04-06 09:45:52,659 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
292
- INFO 2026-04-06 09:48:44,228 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
293
- INFO 2026-04-06 09:51:36,496 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
294
- INFO 2026-04-06 09:54:29,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
295
- INFO 2026-04-06 09:57:20,633 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
296
- INFO 2026-04-06 10:00:22,925 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
297
- INFO 2026-04-06 10:03:48,883 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
298
- INFO 2026-04-06 10:06:52,079 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
299
- INFO 2026-04-06 10:09:59,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
300
- INFO 2026-04-06 10:13:05,838 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
301
- INFO 2026-04-06 10:16:11,930 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
302
- INFO 2026-04-06 10:19:15,974 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
303
- INFO 2026-04-06 10:22:24,663 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
304
- INFO 2026-04-06 10:25:33,547 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
305
- INFO 2026-04-06 10:28:42,171 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
306
- INFO 2026-04-06 10:31:51,514 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
307
- INFO 2026-04-06 10:35:00,444 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
308
- INFO 2026-04-06 10:38:08,605 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
309
- INFO 2026-04-06 10:41:15,596 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
310
- INFO 2026-04-06 11:29:31,530 barrel.train.components.optimizers.optimizer_module init_optimizer rank=7, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
311
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
312
- hyperparams=ConfigDict[typing.Any](
313
- lr=5e-05,
314
- weight_decay=0.0,
315
- ),
316
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
317
- torch_optimizer='torch.optim.AdamW',
318
- ), scheduler=SequentialSchedulerConfig(
319
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
320
- milestones=ConfigList[int](
321
- 500,
322
- ),
323
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
324
- warmup=TorchSchedulerConfig(
325
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
326
- hyperparams=ConfigDict[typing.Any](
327
- start_factor=0.0002,
328
- end_factor=1.0,
329
- total_iters=500,
330
- ),
331
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
332
- ),
333
- const=TorchSchedulerConfig(
334
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
335
- hyperparams=ConfigDict[typing.Any](
336
- factor=1.0,
337
- total_iters=300000,
338
- ),
339
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
340
- ),
341
- ),
342
- )
343
- INFO 2026-04-06 11:29:31,533 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
344
- INFO 2026-04-06 11:29:34,898 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
345
- INFO 2026-04-06 11:29:43,471 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
346
- INFO 2026-04-06 11:29:47,725 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
347
- INFO 2026-04-06 11:29:51,391 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
348
- INFO 2026-04-06 11:29:51,391 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=16493.5, dataset_name=bridge_paraphrase_k5
349
- INFO 2026-04-06 11:29:51,433 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183242, rank=7
350
- INFO 2026-04-06 11:29:52,254 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=7, status=start
351
- INFO 2026-04-06 11:29:52,293 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
352
- INFO 2026-04-06 11:29:52,295 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=20761.7
353
- INFO 2026-04-06 11:29:52,300 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=7, status=start
354
- INFO 2026-04-06 11:31:39,840 barrel.train.core.trainer.trainer Restore session checkpoint rank=7, status=success
355
- INFO 2026-04-06 11:32:01,842 barrel.train.core.trainer.trainer session_init status=success, rank=7
356
- INFO 2026-04-06 11:32:01,844 barrel.train.core.trainer.trainer start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
357
- INFO 2026-04-06 11:32:02,002 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-7, worker_pid=3492776, parent_pid=3489447, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
358
- INFO 2026-04-06 11:32:02,028 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-7, worker_pid=3492784, parent_pid=3489447, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
359
- INFO 2026-04-06 11:35:29,327 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
360
- INFO 2026-04-06 11:38:35,202 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
361
- INFO 2026-04-06 11:41:41,424 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
362
- INFO 2026-04-06 11:44:46,845 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
363
- INFO 2026-04-06 11:47:52,312 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
364
- INFO 2026-04-06 11:50:56,955 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
365
- INFO 2026-04-06 11:54:03,038 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
366
- INFO 2026-04-06 11:57:07,421 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
367
- INFO 2026-04-06 12:00:13,663 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
368
- INFO 2026-04-06 12:03:19,349 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
369
- INFO 2026-04-06 12:06:23,693 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
370
- INFO 2026-04-06 12:09:29,444 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
371
- INFO 2026-04-06 12:12:35,568 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
372
- INFO 2026-04-06 12:15:41,415 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
373
- INFO 2026-04-06 12:18:46,702 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
374
- INFO 2026-04-06 12:21:51,861 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
375
- INFO 2026-04-06 12:24:57,746 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
376
- INFO 2026-04-06 12:28:01,896 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
377
- INFO 2026-04-06 12:31:07,226 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
378
- INFO 2026-04-06 12:34:11,865 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
379
- INFO 2026-04-06 12:37:16,746 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
380
- INFO 2026-04-06 12:40:21,216 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
381
- INFO 2026-04-06 12:43:25,432 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
382
- INFO 2026-04-06 12:46:31,790 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
383
- INFO 2026-04-06 12:49:37,506 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
384
- INFO 2026-04-06 12:52:43,059 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
385
- INFO 2026-04-06 12:55:47,770 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
386
- INFO 2026-04-06 12:58:53,484 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
387
- INFO 2026-04-06 13:01:58,050 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
388
- INFO 2026-04-06 13:05:03,711 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
389
- INFO 2026-04-06 13:08:09,154 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
390
- INFO 2026-04-06 13:11:13,196 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
391
- INFO 2026-04-06 13:14:19,368 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
392
- INFO 2026-04-06 13:17:25,217 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
393
- INFO 2026-04-06 13:20:30,196 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
394
- INFO 2026-04-06 13:23:35,853 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
395
- INFO 2026-04-06 13:26:41,878 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
396
- INFO 2026-04-06 13:29:48,080 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
397
- INFO 2026-04-06 13:32:52,614 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
398
- INFO 2026-04-06 13:35:57,628 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
399
- INFO 2026-04-06 13:39:03,078 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
400
- INFO 2026-04-06 13:42:10,402 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
401
- INFO 2026-04-06 13:45:14,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
402
- INFO 2026-04-06 13:48:20,996 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
403
- INFO 2026-04-06 13:51:25,869 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
404
- INFO 2026-04-06 13:54:30,720 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
405
- INFO 2026-04-06 13:57:36,002 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
406
- INFO 2026-04-06 14:00:41,873 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
407
- INFO 2026-04-06 14:03:49,079 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
408
- INFO 2026-04-06 14:07:04,763 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
409
- INFO 2026-04-06 14:10:39,266 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
410
- INFO 2026-04-06 14:13:44,078 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
411
- INFO 2026-04-06 14:16:49,282 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
412
- INFO 2026-04-06 14:19:53,530 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
413
- INFO 2026-04-06 14:22:58,776 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
414
- INFO 2026-04-06 14:26:03,758 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
415
- INFO 2026-04-06 14:29:07,808 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
416
- INFO 2026-04-06 14:32:11,851 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
417
- INFO 2026-04-06 14:35:16,308 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
418
- INFO 2026-04-06 14:38:22,033 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
419
- INFO 2026-04-06 14:41:25,485 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
420
- INFO 2026-04-06 14:44:30,526 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
421
- INFO 2026-04-06 14:47:36,041 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
422
- INFO 2026-04-06 14:50:40,763 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
423
- INFO 2026-04-06 14:53:45,032 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
424
- INFO 2026-04-06 14:56:50,250 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
425
- INFO 2026-04-06 14:59:56,684 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
426
- INFO 2026-04-06 15:03:02,320 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
427
- INFO 2026-04-07 00:41:05,444 barrel.train.components.optimizers.optimizer_module init_optimizer rank=7, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
428
- autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
429
- hyperparams=ConfigDict[typing.Any](
430
- lr=5e-05,
431
- weight_decay=0.0,
432
- ),
433
- param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
434
- torch_optimizer='torch.optim.AdamW',
435
- ), scheduler=SequentialSchedulerConfig(
436
- autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
437
- milestones=ConfigList[int](
438
- 500,
439
- ),
440
- schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
441
- warmup=TorchSchedulerConfig(
442
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
443
- hyperparams=ConfigDict[typing.Any](
444
- start_factor=0.0002,
445
- end_factor=1.0,
446
- total_iters=500,
447
- ),
448
- torch_scheduler='torch.optim.lr_scheduler.LinearLR',
449
- ),
450
- const=TorchSchedulerConfig(
451
- autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
452
- hyperparams=ConfigDict[typing.Any](
453
- factor=1.0,
454
- total_iters=300000,
455
- ),
456
- torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
457
- ),
458
- ),
459
- )
460
- INFO 2026-04-07 00:41:05,448 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=start
461
- INFO 2026-04-07 00:41:08,589 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=start, dataset_name=bridge_paraphrase_k5
462
- INFO 2026-04-07 00:41:18,091 barrel.train.pipes.vlams.data.robotics.hf.utils.transforms TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
463
- INFO 2026-04-07 00:41:23,206 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
464
- INFO 2026-04-07 00:41:27,430 barrel.train.pipes.vlams.data.robotics.hf.utils.filter Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
465
- INFO 2026-04-07 00:41:27,430 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source apply_transforms status=success, duration_ms=18840.6, dataset_name=bridge_paraphrase_k5
466
- INFO 2026-04-07 00:41:27,471 barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183242, rank=7
467
- INFO 2026-04-07 00:41:28,495 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Computing data pointers for bridge_paraphrase_k5 rank=7, status=start
468
- INFO 2026-04-07 00:41:28,534 barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset Data pointers for bridge_paraphrase_k5 dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
469
- INFO 2026-04-07 00:41:28,536 barrel.train.pipes.vlams.pipe.oxe_data_module load_train_dataset status=success, duration_ms=23088.3
470
- INFO 2026-04-07 00:41:28,541 barrel.train.core.trainer.trainer Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=7, status=start
471
- INFO 2026-04-07 00:43:48,421 barrel.train.core.trainer.trainer Restore session checkpoint rank=7, status=success
472
- INFO 2026-04-07 00:44:10,422 barrel.train.core.trainer.trainer session_init status=success, rank=7
473
- INFO 2026-04-07 00:44:10,423 barrel.train.core.trainer.trainer start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
474
- INFO 2026-04-07 00:44:10,488 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=3313511, parent_pid=3310185, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
475
- INFO 2026-04-07 00:44:10,525 barrel.train.components.data.torch_dataloader Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=3313538, parent_pid=3310185, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
476
- INFO 2026-04-07 00:47:36,031 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
477
- INFO 2026-04-07 00:50:38,874 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
478
- INFO 2026-04-07 00:53:42,316 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
479
- INFO 2026-04-07 00:56:45,089 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
480
- INFO 2026-04-07 00:59:47,992 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
481
- INFO 2026-04-07 01:02:50,213 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
482
- INFO 2026-04-07 01:05:53,531 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
483
- INFO 2026-04-07 01:08:56,073 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
484
- INFO 2026-04-07 01:11:58,579 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
485
- INFO 2026-04-07 01:15:02,216 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
486
- INFO 2026-04-07 01:18:05,106 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
487
- INFO 2026-04-07 01:21:08,476 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
488
- INFO 2026-04-07 01:24:11,705 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
489
- INFO 2026-04-07 01:27:15,146 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
490
- INFO 2026-04-07 01:30:16,888 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
491
- INFO 2026-04-07 01:33:19,438 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
492
- INFO 2026-04-07 01:36:22,459 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
493
- INFO 2026-04-07 01:39:25,147 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
494
- INFO 2026-04-07 01:42:29,626 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
495
- INFO 2026-04-07 01:45:32,639 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
496
- INFO 2026-04-07 01:48:36,841 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
497
- INFO 2026-04-07 01:51:39,495 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
498
- INFO 2026-04-07 01:54:42,918 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
499
- INFO 2026-04-07 01:57:46,413 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
500
- INFO 2026-04-07 02:00:50,175 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
501
- INFO 2026-04-07 02:03:54,285 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
502
- INFO 2026-04-07 02:06:57,978 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
503
- INFO 2026-04-07 02:10:02,509 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
504
- INFO 2026-04-07 02:13:05,977 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
505
- INFO 2026-04-07 02:16:10,143 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
506
- INFO 2026-04-07 02:19:14,244 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
507
- INFO 2026-04-07 02:22:17,143 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
508
- INFO 2026-04-07 02:25:21,418 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
509
- INFO 2026-04-07 02:28:24,740 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
510
- INFO 2026-04-07 02:31:29,710 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
511
- INFO 2026-04-07 02:34:33,439 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
512
- INFO 2026-04-07 02:37:37,745 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
513
- INFO 2026-04-07 02:40:41,393 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
514
- INFO 2026-04-07 02:43:44,857 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
515
- INFO 2026-04-07 02:46:48,390 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
516
- INFO 2026-04-07 02:49:52,448 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
517
- INFO 2026-04-07 02:52:56,836 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
518
- INFO 2026-04-07 02:56:01,914 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
519
- INFO 2026-04-07 02:59:07,333 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
520
- INFO 2026-04-07 03:02:11,728 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
521
- INFO 2026-04-07 03:05:16,730 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
522
- INFO 2026-04-07 03:08:20,104 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
523
- INFO 2026-04-07 03:11:23,983 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
524
- INFO 2026-04-07 03:14:28,690 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
525
- INFO 2026-04-07 03:17:51,497 barrel.train.pipes.vlams.pipe.robotics_vis_callback visualize_train_data status=model generate
526
- INFO 2026-04-07 03:18:34,222 barrel.train.core.trainer.trainer training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=7
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d91786f01dd1b0812225c748c18bb066833412534bd95d24015d446350ba4ae4
3
+ size 64010