backup: add missing files to sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
Browse files- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._max_dataframe.__.parquet +3 -0
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._min_dataframe.__.parquet +3 -0
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._rand_dataframe.__.parquet +3 -0
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/config.yaml +3 -215
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/full_config.yaml +3 -496
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/format.log +3 -24
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/model_config.yaml +3 -120
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/language_results.json +3 -0
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/model_config.yaml +3 -137
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/raw_config.yaml +3 -11
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session.log +3 -0
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_info.json +3 -1
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_1.log +3 -526
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_2.log +3 -526
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_3.log +3 -526
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_4.log +3 -526
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_5.log +3 -526
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_6.log +3 -526
- sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_7.log +3 -526
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._max_dataframe.__.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0ad9c0c47c0e592e937c931759cbc06d797fcb0316b5bc9ce1873a1ef5c00bd
|
| 3 |
+
size 985
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._min_dataframe.__.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0ad9c0c47c0e592e937c931759cbc06d797fcb0316b5bc9ce1873a1ef5c00bd
|
| 3 |
+
size 985
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-030000-data/callbacks.visualization_callback._rand_dataframe.__.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0ad9c0c47c0e592e937c931759cbc06d797fcb0316b5bc9ce1873a1ef5c00bd
|
| 3 |
+
size 985
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/config.yaml
CHANGED
|
@@ -1,215 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
metrics_config:
|
| 5 |
-
buckets:
|
| 6 |
-
wandb:
|
| 7 |
-
autoclass: barrel.train.core.trainer.metrics_buckets.wandb_metrics_bucket.WandbMetricsBucket
|
| 8 |
-
project: diffusion_vlams
|
| 9 |
-
gpus: 8
|
| 10 |
-
num_steps: 15000
|
| 11 |
-
test_interval: 10000
|
| 12 |
-
test_enabled: false
|
| 13 |
-
training_module_config:
|
| 14 |
-
autoclass: barrel.train.pipes.vlams.pipe.vlam_training_module.VLAMTrainingModule
|
| 15 |
-
grad_accumulation_steps: 2
|
| 16 |
-
autocast_dtype: bfloat16
|
| 17 |
-
fsdp_config:
|
| 18 |
-
autoclass: barrel.train.core.common.torch_utils.fsdp.FSDP2
|
| 19 |
-
enabled: true
|
| 20 |
-
param_dtype: bfloat16
|
| 21 |
-
model_config:
|
| 22 |
-
autoclass: barrel.train.pipes.vlams.models.vlams.vlam.VLAM
|
| 23 |
-
processor_config:
|
| 24 |
-
autoclass: barrel.train.pipes.vlams.types.vlam_processors.PiZeroFlowMatchingProcessor
|
| 25 |
-
control_io_config:
|
| 26 |
-
future_controls_sequence_length: 5
|
| 27 |
-
future_controls_sequence_stride_sec: 0.2
|
| 28 |
-
rotation_format: quaternion
|
| 29 |
-
translation_obs_norm:
|
| 30 |
-
autoclass: barrel.train.components.normalization.normalizers.DatasetStatsNormalizer
|
| 31 |
-
component_name: translation
|
| 32 |
-
mode: bounds_q99
|
| 33 |
-
stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
|
| 34 |
-
rotation_obs_norm:
|
| 35 |
-
autoclass: barrel.train.components.normalization.normalizers.IdentityNormalizer
|
| 36 |
-
translation_control_norm:
|
| 37 |
-
autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
|
| 38 |
-
low:
|
| 39 |
-
- -0.04
|
| 40 |
-
- -0.04
|
| 41 |
-
- -0.04
|
| 42 |
-
high:
|
| 43 |
-
- 0.04
|
| 44 |
-
- 0.04
|
| 45 |
-
- 0.04
|
| 46 |
-
rotation_control_norm:
|
| 47 |
-
autoclass: barrel.train.components.normalization.normalizers.RotationStereomapNormalizer
|
| 48 |
-
factor: 4.0
|
| 49 |
-
control_tokenizer_config:
|
| 50 |
-
autoclass: barrel.train.pipes.vlams.types.action_tokenizer.EmptyTokenizer
|
| 51 |
-
num_inference_steps: 10
|
| 52 |
-
r0_distribution: uniform
|
| 53 |
-
timestep_distribution: beta
|
| 54 |
-
distribution_hyperparams:
|
| 55 |
-
alpha: 1.5
|
| 56 |
-
beta: 1.0
|
| 57 |
-
sig_min: 0.001
|
| 58 |
-
image_resize: naive
|
| 59 |
-
vlm_config:
|
| 60 |
-
autoclass: barrel.train.pipes.vlams.models.vlms.paligemma.PaliGemmaVLM
|
| 61 |
-
model_id: google/paligemma-3b-mix-224
|
| 62 |
-
processor_config:
|
| 63 |
-
autoclass: barrel.train.pipes.vlams.types.vlm_processors.PaliGemmaProcessor
|
| 64 |
-
image_sizes:
|
| 65 |
-
main:
|
| 66 |
-
width: 224
|
| 67 |
-
height: 224
|
| 68 |
-
pretrain_config:
|
| 69 |
-
trainable: true
|
| 70 |
-
frozen_dtype: bfloat16
|
| 71 |
-
state_dict:
|
| 72 |
-
checkpoint_path: PRETRAINED_MODULE
|
| 73 |
-
control_module_config:
|
| 74 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_module.PiZeroFlowMatchingModule
|
| 75 |
-
token_size: 1024
|
| 76 |
-
rotation_components: 4
|
| 77 |
-
noised_control_proj_config:
|
| 78 |
-
autoclass: barrel.train.pipes.vlams.models.control.token_proj.NoisedControlProjector
|
| 79 |
-
time_embed:
|
| 80 |
-
autoclass: barrel.train.components.nn.layers.fourier_features.FourierFeatures
|
| 81 |
-
num_features: 1024
|
| 82 |
-
layers: []
|
| 83 |
-
max_period: 10000.0
|
| 84 |
-
layers:
|
| 85 |
-
- 8
|
| 86 |
-
- 2048
|
| 87 |
-
- 1024
|
| 88 |
-
- 1024
|
| 89 |
-
activation: SiLU
|
| 90 |
-
robot_state_proj_config:
|
| 91 |
-
autoclass: barrel.train.pipes.vlams.models.control.token_proj.RobotStateProjector
|
| 92 |
-
layers:
|
| 93 |
-
- 8
|
| 94 |
-
- 1024
|
| 95 |
-
mode: ee_pose_gripper
|
| 96 |
-
activation: SiLU
|
| 97 |
-
control_decoder_config:
|
| 98 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoder
|
| 99 |
-
num_blocks: 18
|
| 100 |
-
block_config:
|
| 101 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoderBlock
|
| 102 |
-
feature_size: 1024
|
| 103 |
-
head_dim: 256
|
| 104 |
-
num_heads: 8
|
| 105 |
-
num_kv_heads: 1
|
| 106 |
-
hidden_size: 4096
|
| 107 |
-
activation: GELU
|
| 108 |
-
activation_kwargs:
|
| 109 |
-
approximate: tanh
|
| 110 |
-
attn_implementation: sdpa
|
| 111 |
-
position_embed_config:
|
| 112 |
-
autoclass: barrel.train.components.nn.layers.positional_encodings.RotaryPositionalEncoding
|
| 113 |
-
num_embeddings: 512
|
| 114 |
-
embedding_dim: 256
|
| 115 |
-
base: 10000
|
| 116 |
-
loss_config:
|
| 117 |
-
autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRoboticsLoss
|
| 118 |
-
discount: false
|
| 119 |
-
rotation_fm_loss: cosine_similarity
|
| 120 |
-
rotation_loss:
|
| 121 |
-
autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRotationLoss
|
| 122 |
-
method: integrate
|
| 123 |
-
step_size_range:
|
| 124 |
-
- 0.01
|
| 125 |
-
- 1.0
|
| 126 |
-
rotation_losses:
|
| 127 |
-
geodesic:
|
| 128 |
-
autoclass: barrel.train.pipes.vlams.losses.rotation_loss.RotationLoss
|
| 129 |
-
loss_format: geodesic
|
| 130 |
-
power: 2
|
| 131 |
-
llm_loss:
|
| 132 |
-
autoclass: barrel.train.pipes.vlams.losses.vlm_loss.VLMLoss
|
| 133 |
-
weights:
|
| 134 |
-
translation_loss: 1.0
|
| 135 |
-
rotation_loss: 1.0
|
| 136 |
-
gripper_loss: 1.0
|
| 137 |
-
llm_loss: 0.0
|
| 138 |
-
waypoints_loss: 1.0
|
| 139 |
-
optimizer_config:
|
| 140 |
-
optimizer:
|
| 141 |
-
autoclass: barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer
|
| 142 |
-
torch_optimizer: torch.optim.AdamW
|
| 143 |
-
hyperparams:
|
| 144 |
-
lr: 5.0e-05
|
| 145 |
-
weight_decay: 0.0
|
| 146 |
-
scheduler:
|
| 147 |
-
autoclass: barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler
|
| 148 |
-
schedulers:
|
| 149 |
-
warmup:
|
| 150 |
-
autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
|
| 151 |
-
torch_scheduler: torch.optim.lr_scheduler.LinearLR
|
| 152 |
-
hyperparams:
|
| 153 |
-
start_factor: 0.0002
|
| 154 |
-
end_factor: 1.0
|
| 155 |
-
total_iters: 500
|
| 156 |
-
const:
|
| 157 |
-
autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
|
| 158 |
-
torch_scheduler: torch.optim.lr_scheduler.ConstantLR
|
| 159 |
-
hyperparams:
|
| 160 |
-
factor: 1.0
|
| 161 |
-
total_iters: 300000
|
| 162 |
-
milestones:
|
| 163 |
-
- 500
|
| 164 |
-
clip_grad_norm: 1.0
|
| 165 |
-
data_module_config:
|
| 166 |
-
autoclass: barrel.train.pipes.vlams.pipe.oxe_data_module.OXEDataModule
|
| 167 |
-
dataset_configs:
|
| 168 |
-
bridge:
|
| 169 |
-
autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset.RoboticsDataset
|
| 170 |
-
data_source_config:
|
| 171 |
-
autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source.RoboticsDataSource
|
| 172 |
-
dataset_path: /home/petko_petkov/datasets/oxe/lerobot-video/bridge
|
| 173 |
-
stacker_config:
|
| 174 |
-
remove_invalid_future_pointers: true
|
| 175 |
-
dataset_weights:
|
| 176 |
-
bridge: 1.0
|
| 177 |
-
batch_size: 1024
|
| 178 |
-
callbacks_config:
|
| 179 |
-
callbacks:
|
| 180 |
-
params_callback:
|
| 181 |
-
autoclass: barrel.train.components.callbacks.params_callback.ParametersCallback
|
| 182 |
-
progressbar_callback:
|
| 183 |
-
autoclass: barrel.train.components.callbacks.progressbar_callback.ProgressbarCallback
|
| 184 |
-
print_callback:
|
| 185 |
-
autoclass: barrel.train.components.callbacks.trainer_monitor_callback.TrainerMonitorCallback
|
| 186 |
-
print_interval: 100
|
| 187 |
-
debug_nans_callback:
|
| 188 |
-
autoclass: barrel.train.components.callbacks.debug_nans_callback.DebugNansCallback
|
| 189 |
-
enabled: true
|
| 190 |
-
plot_grads_callback:
|
| 191 |
-
autoclass: barrel.train.components.callbacks.plot_grads_callback.PlotGradsCallback
|
| 192 |
-
plot_interval: 200
|
| 193 |
-
enabled: false
|
| 194 |
-
checkpoint_callback:
|
| 195 |
-
autoclass: barrel.train.components.callbacks.checkpoint_callback.CheckpointCallback
|
| 196 |
-
session_save_interval: 5000
|
| 197 |
-
keep_test_checkpoints: false
|
| 198 |
-
keep_trainer_steps: []
|
| 199 |
-
keep_model_steps:
|
| 200 |
-
- 15000
|
| 201 |
-
ema_config:
|
| 202 |
-
enabled: true
|
| 203 |
-
decay: 0.99
|
| 204 |
-
interval: 1
|
| 205 |
-
start_step: 20000
|
| 206 |
-
keep_trainer_percent: []
|
| 207 |
-
keep_model_percent: []
|
| 208 |
-
visualization_callback:
|
| 209 |
-
autoclass: barrel.train.pipes.vlams.pipe.robotics_vis_callback.RoboticsVisualizationCallback
|
| 210 |
-
visualization_interval: 100
|
| 211 |
-
inference_export_callback:
|
| 212 |
-
autoclass: barrel.train.components.callbacks.inference_export_callback.InferenceExportCallback
|
| 213 |
-
model_name: pizero_fm_paligemma
|
| 214 |
-
test_callback:
|
| 215 |
-
autoclass: barrel.train.pipes.vlams.pipe.robotics_test_dataset_callback.RoboticsTestDatasetCallback
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:06ea035872d31218c768fdd4506d7f98c7ac60449769a838d5821161c8f61b65
|
| 3 |
+
size 7921
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/full_config.yaml
CHANGED
|
@@ -1,496 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
params_callback:
|
| 5 |
-
autoclass: barrel.train.components.callbacks.params_callback.ParametersCallback
|
| 6 |
-
enabled: true
|
| 7 |
-
progressbar_callback:
|
| 8 |
-
autoclass: barrel.train.components.callbacks.progressbar_callback.ProgressbarCallback
|
| 9 |
-
enabled: true
|
| 10 |
-
print_callback:
|
| 11 |
-
autoclass: barrel.train.components.callbacks.trainer_monitor_callback.TrainerMonitorCallback
|
| 12 |
-
print_interval: 100
|
| 13 |
-
debug_nans_callback:
|
| 14 |
-
autoclass: barrel.train.components.callbacks.debug_nans_callback.DebugNansCallback
|
| 15 |
-
check_batch: false
|
| 16 |
-
check_grads: false
|
| 17 |
-
check_loss: true
|
| 18 |
-
check_output: false
|
| 19 |
-
check_params: false
|
| 20 |
-
drop_to_pdb: false
|
| 21 |
-
enabled: true
|
| 22 |
-
plot_grads_callback:
|
| 23 |
-
autoclass: barrel.train.components.callbacks.plot_grads_callback.PlotGradsCallback
|
| 24 |
-
enabled: false
|
| 25 |
-
names: []
|
| 26 |
-
plot_interval: 200
|
| 27 |
-
checkpoint_callback:
|
| 28 |
-
autoclass: barrel.train.components.callbacks.checkpoint_callback.CheckpointCallback
|
| 29 |
-
ema_config:
|
| 30 |
-
decay: 0.99
|
| 31 |
-
enabled: true
|
| 32 |
-
interval: 1
|
| 33 |
-
start_step: 20000
|
| 34 |
-
enabled: true
|
| 35 |
-
keep_model_percent: []
|
| 36 |
-
keep_model_steps:
|
| 37 |
-
- 15000
|
| 38 |
-
keep_test_checkpoints: false
|
| 39 |
-
keep_trainer_percent: []
|
| 40 |
-
keep_trainer_steps: []
|
| 41 |
-
save_at_init: false
|
| 42 |
-
separate_model: true
|
| 43 |
-
session_save_interval: 5000
|
| 44 |
-
visualization_callback:
|
| 45 |
-
autoclass: barrel.train.pipes.vlams.pipe.robotics_vis_callback.RoboticsVisualizationCallback
|
| 46 |
-
plot_table: false
|
| 47 |
-
token_accuracy_metrics: false
|
| 48 |
-
visualization_interval: 100
|
| 49 |
-
inference_export_callback:
|
| 50 |
-
autoclass: barrel.train.components.callbacks.inference_export_callback.InferenceExportCallback
|
| 51 |
-
databib_enabled: true
|
| 52 |
-
enabled: true
|
| 53 |
-
model_name: pizero_fm_paligemma
|
| 54 |
-
test_callback:
|
| 55 |
-
autoclass: barrel.train.pipes.vlams.pipe.robotics_test_dataset_callback.RoboticsTestDatasetCallback
|
| 56 |
-
batch_size: null
|
| 57 |
-
dataloader_workers: 2
|
| 58 |
-
dataset_configs: {}
|
| 59 |
-
enabled: true
|
| 60 |
-
max_batches: null
|
| 61 |
-
data_module_config:
|
| 62 |
-
autoclass: barrel.train.pipes.vlams.pipe.oxe_data_module.OXEDataModule
|
| 63 |
-
batch_size: 1024
|
| 64 |
-
dataloader_workers: 2
|
| 65 |
-
dataset_configs:
|
| 66 |
-
bridge:
|
| 67 |
-
augment_kwargs:
|
| 68 |
-
main:
|
| 69 |
-
color_jitter:
|
| 70 |
-
brightness:
|
| 71 |
-
- 0.9
|
| 72 |
-
- 1.1
|
| 73 |
-
contrast:
|
| 74 |
-
- 0.9
|
| 75 |
-
- 1.1
|
| 76 |
-
saturation:
|
| 77 |
-
- 0.9
|
| 78 |
-
- 1.1
|
| 79 |
-
hue: 0.05
|
| 80 |
-
resize:
|
| 81 |
-
scale:
|
| 82 |
-
- 0.8
|
| 83 |
-
- 1.0
|
| 84 |
-
ratio:
|
| 85 |
-
- 0.1
|
| 86 |
-
- 0.1
|
| 87 |
-
wrist:
|
| 88 |
-
color_jitter:
|
| 89 |
-
brightness:
|
| 90 |
-
- 0.9
|
| 91 |
-
- 1.1
|
| 92 |
-
contrast:
|
| 93 |
-
- 0.9
|
| 94 |
-
- 1.1
|
| 95 |
-
saturation:
|
| 96 |
-
- 0.9
|
| 97 |
-
- 1.1
|
| 98 |
-
hue: 0.05
|
| 99 |
-
resize:
|
| 100 |
-
scale:
|
| 101 |
-
- 0.8
|
| 102 |
-
- 1.0
|
| 103 |
-
ratio:
|
| 104 |
-
- 0.1
|
| 105 |
-
- 0.1
|
| 106 |
-
autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_dataset.RoboticsDataset
|
| 107 |
-
axes_augment_prob: 0.0
|
| 108 |
-
balancer_config: null
|
| 109 |
-
control_augment_config:
|
| 110 |
-
future_controls_augment_window: null
|
| 111 |
-
future_frames_augment_window: null
|
| 112 |
-
past_frames_augment_window: null
|
| 113 |
-
past_scalars_augment_window: null
|
| 114 |
-
corrective_augment_prob: 0.0
|
| 115 |
-
data_source_config:
|
| 116 |
-
autoclass: barrel.train.pipes.vlams.data.robotics.hf.robotics_data_source.RoboticsDataSource
|
| 117 |
-
dataset_path: /home/petko_petkov/datasets/oxe/lerobot-video/bridge
|
| 118 |
-
df_transform_config:
|
| 119 |
-
drop_columns: []
|
| 120 |
-
episode_length_threshold: null
|
| 121 |
-
filter_static_ee_frames: true
|
| 122 |
-
gripper_threshold: auto
|
| 123 |
-
min_episode_displacement_m: 0.05
|
| 124 |
-
remap_columns:
|
| 125 |
-
observation.images.image_0: observation.images.main_image
|
| 126 |
-
observation.images.image_3: observation.images.wrist_image
|
| 127 |
-
rotation_threshold_deg: 1.0
|
| 128 |
-
transform_kwargs:
|
| 129 |
-
augment: false
|
| 130 |
-
env_regex: ''
|
| 131 |
-
translation_threshold_m: 0.005
|
| 132 |
-
skip_no_language: true
|
| 133 |
-
split: train[:100%]
|
| 134 |
-
dataset_mode: train
|
| 135 |
-
num_image_reader_threads: 2
|
| 136 |
-
stacker_config:
|
| 137 |
-
future_controls_shift_sec: 0.0
|
| 138 |
-
future_frames_shift_sec: 0.0
|
| 139 |
-
min_future_horizon_sec: null
|
| 140 |
-
min_past_horizon_sec: null
|
| 141 |
-
past_frames_shift_sec: 0.0
|
| 142 |
-
past_scalars_shift_sec: 0.0
|
| 143 |
-
remove_invalid_future_pointers: true
|
| 144 |
-
remove_invalid_past_pointers: true
|
| 145 |
-
row_skip_offset: 0
|
| 146 |
-
row_skip_stride: 1
|
| 147 |
-
timestamp_max_gap_sec: 0.04
|
| 148 |
-
timestamp_precision_sec: 0.005
|
| 149 |
-
total_rows: -1
|
| 150 |
-
dataset_weights:
|
| 151 |
-
bridge: 1.0
|
| 152 |
-
debug_mode: false
|
| 153 |
-
deterministic: false
|
| 154 |
-
gpus: 8
|
| 155 |
-
num_steps: 15000
|
| 156 |
-
seed: 1222297665
|
| 157 |
-
session_config:
|
| 158 |
-
config_path: barrel/train/pipes/vlams/configs/control/pizero_fm_paligemma_bridge_curriculum_phase1.yaml
|
| 159 |
-
extra_command_line_args: {}
|
| 160 |
-
metrics_config:
|
| 161 |
-
buckets:
|
| 162 |
-
wandb:
|
| 163 |
-
autoclass: barrel.train.core.trainer.metrics_buckets.wandb_metrics_bucket.WandbMetricsBucket
|
| 164 |
-
enabled: true
|
| 165 |
-
fork_from: ''
|
| 166 |
-
project: diffusion_vlams
|
| 167 |
-
tags: []
|
| 168 |
-
enabled: true
|
| 169 |
-
tags: []
|
| 170 |
-
output_path: /home/petko_petkov/experiments/vlams/control
|
| 171 |
-
restore_path: null
|
| 172 |
-
session_name: sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 173 |
-
session_path: /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 174 |
-
tag: bridge_curriculum_phase1
|
| 175 |
-
test_enabled: false
|
| 176 |
-
test_interval: 10000
|
| 177 |
-
training_module_config:
|
| 178 |
-
autocast_dtype: bfloat16
|
| 179 |
-
autocast_enabled: true
|
| 180 |
-
autoclass: barrel.train.pipes.vlams.pipe.vlam_training_module.VLAMTrainingModule
|
| 181 |
-
fsdp_config:
|
| 182 |
-
autoclass: barrel.train.core.common.torch_utils.fsdp.FSDP2
|
| 183 |
-
cast_forward_inputs: true
|
| 184 |
-
enabled: true
|
| 185 |
-
output_dtype: null
|
| 186 |
-
param_dtype: bfloat16
|
| 187 |
-
reduce_dtype: float32
|
| 188 |
-
reshard_after_forward: true
|
| 189 |
-
grad_accumulation_steps: 2
|
| 190 |
-
loss_config:
|
| 191 |
-
autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRoboticsLoss
|
| 192 |
-
discount: false
|
| 193 |
-
discount_scale_param: 1
|
| 194 |
-
discount_shape_param: 1
|
| 195 |
-
gripper_fm_loss: mse_loss
|
| 196 |
-
llm_loss:
|
| 197 |
-
autoclass: barrel.train.pipes.vlams.losses.vlm_loss.VLMLoss
|
| 198 |
-
weight: 1.0
|
| 199 |
-
weight_ids: null
|
| 200 |
-
num_timesteps: null
|
| 201 |
-
rotation_fm_loss: cosine_similarity
|
| 202 |
-
rotation_loss:
|
| 203 |
-
autoclass: barrel.train.pipes.vlams.losses.flow_matching_loss.FlowMatchingRotationLoss
|
| 204 |
-
method: integrate
|
| 205 |
-
rotation_losses:
|
| 206 |
-
geodesic:
|
| 207 |
-
autoclass: barrel.train.pipes.vlams.losses.rotation_loss.RotationLoss
|
| 208 |
-
loss_format: geodesic
|
| 209 |
-
power: 2
|
| 210 |
-
rotation_format: ''
|
| 211 |
-
step_size_range:
|
| 212 |
-
- 0.01
|
| 213 |
-
- 1.0
|
| 214 |
-
time_delta_sec: null
|
| 215 |
-
translation_fm_loss: mse_loss
|
| 216 |
-
weights:
|
| 217 |
-
translation_loss: 1.0
|
| 218 |
-
rotation_loss: 1.0
|
| 219 |
-
gripper_loss: 1.0
|
| 220 |
-
llm_loss: 0.0
|
| 221 |
-
waypoints_loss: 1.0
|
| 222 |
-
model_config:
|
| 223 |
-
autoclass: barrel.train.pipes.vlams.models.vlams.vlam.VLAM
|
| 224 |
-
control_module_config:
|
| 225 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_module.PiZeroFlowMatchingModule
|
| 226 |
-
control_decoder_config:
|
| 227 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoder
|
| 228 |
-
block_config:
|
| 229 |
-
activation: GELU
|
| 230 |
-
activation_kwargs:
|
| 231 |
-
approximate: tanh
|
| 232 |
-
attn_implementation: sdpa
|
| 233 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoderBlock
|
| 234 |
-
dropout: 0.0
|
| 235 |
-
feature_size: 1024
|
| 236 |
-
head_dim: 256
|
| 237 |
-
hidden_size: 4096
|
| 238 |
-
norm: RMSNorm
|
| 239 |
-
num_heads: 8
|
| 240 |
-
num_kv_heads: 1
|
| 241 |
-
position_embed_config:
|
| 242 |
-
autoclass: barrel.train.components.nn.layers.positional_encodings.RotaryPositionalEncoding
|
| 243 |
-
base: 10000
|
| 244 |
-
cached: true
|
| 245 |
-
embedding_dim: 256
|
| 246 |
-
num_embeddings: 512
|
| 247 |
-
pretrain_config:
|
| 248 |
-
exclude_modules: []
|
| 249 |
-
fine_tuneable_modules: []
|
| 250 |
-
frozen_dtype: float32
|
| 251 |
-
state_dict:
|
| 252 |
-
checkpoint_path: null
|
| 253 |
-
drop_keys: []
|
| 254 |
-
hf_filename: null
|
| 255 |
-
hf_hub_repo: null
|
| 256 |
-
load_key: ''
|
| 257 |
-
remap_keys: {}
|
| 258 |
-
trainable: null
|
| 259 |
-
pretrain_config:
|
| 260 |
-
exclude_modules: []
|
| 261 |
-
fine_tuneable_modules: []
|
| 262 |
-
frozen_dtype: float32
|
| 263 |
-
state_dict:
|
| 264 |
-
checkpoint_path: null
|
| 265 |
-
drop_keys: []
|
| 266 |
-
hf_filename: null
|
| 267 |
-
hf_hub_repo: null
|
| 268 |
-
load_key: ''
|
| 269 |
-
remap_keys: {}
|
| 270 |
-
trainable: null
|
| 271 |
-
num_blocks: 18
|
| 272 |
-
pretrain_config:
|
| 273 |
-
exclude_modules: []
|
| 274 |
-
fine_tuneable_modules: []
|
| 275 |
-
frozen_dtype: float32
|
| 276 |
-
state_dict:
|
| 277 |
-
checkpoint_path: null
|
| 278 |
-
drop_keys: []
|
| 279 |
-
hf_filename: null
|
| 280 |
-
hf_hub_repo: null
|
| 281 |
-
load_key: ''
|
| 282 |
-
remap_keys: {}
|
| 283 |
-
trainable: null
|
| 284 |
-
noised_control_proj_config:
|
| 285 |
-
activation: SiLU
|
| 286 |
-
autoclass: barrel.train.pipes.vlams.models.control.token_proj.NoisedControlProjector
|
| 287 |
-
layers:
|
| 288 |
-
- 8
|
| 289 |
-
- 2048
|
| 290 |
-
- 1024
|
| 291 |
-
- 1024
|
| 292 |
-
norm: null
|
| 293 |
-
pretrain_config:
|
| 294 |
-
exclude_modules: []
|
| 295 |
-
fine_tuneable_modules: []
|
| 296 |
-
frozen_dtype: float32
|
| 297 |
-
state_dict:
|
| 298 |
-
checkpoint_path: null
|
| 299 |
-
drop_keys: []
|
| 300 |
-
hf_filename: null
|
| 301 |
-
hf_hub_repo: null
|
| 302 |
-
load_key: ''
|
| 303 |
-
remap_keys: {}
|
| 304 |
-
trainable: null
|
| 305 |
-
time_embed:
|
| 306 |
-
activation: SiLU
|
| 307 |
-
autoclass: barrel.train.components.nn.layers.fourier_features.FourierFeatures
|
| 308 |
-
layers: []
|
| 309 |
-
learnable_features: false
|
| 310 |
-
max_period: 10000.0
|
| 311 |
-
norm: null
|
| 312 |
-
num_features: 1024
|
| 313 |
-
pretrain_config:
|
| 314 |
-
exclude_modules: []
|
| 315 |
-
fine_tuneable_modules: []
|
| 316 |
-
frozen_dtype: float32
|
| 317 |
-
state_dict:
|
| 318 |
-
checkpoint_path: null
|
| 319 |
-
drop_keys: []
|
| 320 |
-
hf_filename: null
|
| 321 |
-
hf_hub_repo: null
|
| 322 |
-
load_key: ''
|
| 323 |
-
remap_keys: {}
|
| 324 |
-
trainable: null
|
| 325 |
-
pretrain_config:
|
| 326 |
-
exclude_modules: []
|
| 327 |
-
fine_tuneable_modules: []
|
| 328 |
-
frozen_dtype: float32
|
| 329 |
-
state_dict:
|
| 330 |
-
checkpoint_path: null
|
| 331 |
-
drop_keys: []
|
| 332 |
-
hf_filename: null
|
| 333 |
-
hf_hub_repo: null
|
| 334 |
-
load_key: ''
|
| 335 |
-
remap_keys: {}
|
| 336 |
-
trainable: null
|
| 337 |
-
robot_state_proj_config:
|
| 338 |
-
activation: SiLU
|
| 339 |
-
autoclass: barrel.train.pipes.vlams.models.control.token_proj.RobotStateProjector
|
| 340 |
-
fourier: false
|
| 341 |
-
layers:
|
| 342 |
-
- 8
|
| 343 |
-
- 1024
|
| 344 |
-
mode: ee_pose_gripper
|
| 345 |
-
pretrain_config:
|
| 346 |
-
exclude_modules: []
|
| 347 |
-
fine_tuneable_modules: []
|
| 348 |
-
frozen_dtype: float32
|
| 349 |
-
state_dict:
|
| 350 |
-
checkpoint_path: null
|
| 351 |
-
drop_keys: []
|
| 352 |
-
hf_filename: null
|
| 353 |
-
hf_hub_repo: null
|
| 354 |
-
load_key: ''
|
| 355 |
-
remap_keys: {}
|
| 356 |
-
trainable: null
|
| 357 |
-
rotation_components: 4
|
| 358 |
-
token_size: 1024
|
| 359 |
-
pretrain_config:
|
| 360 |
-
exclude_modules: []
|
| 361 |
-
fine_tuneable_modules: []
|
| 362 |
-
frozen_dtype: float32
|
| 363 |
-
state_dict:
|
| 364 |
-
checkpoint_path: null
|
| 365 |
-
drop_keys: []
|
| 366 |
-
hf_filename: null
|
| 367 |
-
hf_hub_repo: null
|
| 368 |
-
load_key: ''
|
| 369 |
-
remap_keys: {}
|
| 370 |
-
trainable: null
|
| 371 |
-
processor_config:
|
| 372 |
-
autoclass: barrel.train.pipes.vlams.types.vlam_processors.PiZeroFlowMatchingProcessor
|
| 373 |
-
control_io_config:
|
| 374 |
-
future_control_offset_sec: 0.0
|
| 375 |
-
future_controls_sequence_length: 5
|
| 376 |
-
future_controls_sequence_stride_sec: 0.2
|
| 377 |
-
future_frames_sequence_length: 1
|
| 378 |
-
future_frames_sequence_stride_sec: null
|
| 379 |
-
past_frames_sequence_length: 1
|
| 380 |
-
past_frames_stride_sec: null
|
| 381 |
-
past_scalars_sequence_length: 1
|
| 382 |
-
past_scalars_stride_sec: null
|
| 383 |
-
sequence_frames: 1
|
| 384 |
-
sequence_frames_stride_sec: null
|
| 385 |
-
control_tokenizer_config:
|
| 386 |
-
autoclass: barrel.train.pipes.vlams.types.action_tokenizer.EmptyTokenizer
|
| 387 |
-
distribution_hyperparams:
|
| 388 |
-
alpha: 1.5
|
| 389 |
-
beta: 1.0
|
| 390 |
-
image_resize: naive
|
| 391 |
-
joints_obs_norm:
|
| 392 |
-
autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
|
| 393 |
-
high:
|
| 394 |
-
- 3.141592653589793
|
| 395 |
-
- 3.141592653589793
|
| 396 |
-
- 3.141592653589793
|
| 397 |
-
- 3.141592653589793
|
| 398 |
-
- 3.141592653589793
|
| 399 |
-
- 3.141592653589793
|
| 400 |
-
- 3.141592653589793
|
| 401 |
-
low:
|
| 402 |
-
- -3.141592653589793
|
| 403 |
-
- -3.141592653589793
|
| 404 |
-
- -3.141592653589793
|
| 405 |
-
- -3.141592653589793
|
| 406 |
-
- -3.141592653589793
|
| 407 |
-
- -3.141592653589793
|
| 408 |
-
- -3.141592653589793
|
| 409 |
-
num_inference_steps: 10
|
| 410 |
-
r0_distribution: uniform
|
| 411 |
-
rotation_control_frame: eef_delta
|
| 412 |
-
rotation_control_norm:
|
| 413 |
-
autoclass: barrel.train.components.normalization.normalizers.RotationStereomapNormalizer
|
| 414 |
-
factor: 4.0
|
| 415 |
-
rotation_format: quaternion
|
| 416 |
-
rotation_obs_frame: robot_base
|
| 417 |
-
rotation_obs_norm:
|
| 418 |
-
autoclass: barrel.train.components.normalization.normalizers.IdentityNormalizer
|
| 419 |
-
sig_min: 0.001
|
| 420 |
-
timestep_distribution: beta
|
| 421 |
-
translation_control_frame: robot_base_delta
|
| 422 |
-
translation_control_norm:
|
| 423 |
-
autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
|
| 424 |
-
high:
|
| 425 |
-
- 0.04
|
| 426 |
-
- 0.04
|
| 427 |
-
- 0.04
|
| 428 |
-
low:
|
| 429 |
-
- -0.04
|
| 430 |
-
- -0.04
|
| 431 |
-
- -0.04
|
| 432 |
-
translation_obs_frame: robot_base
|
| 433 |
-
translation_obs_norm:
|
| 434 |
-
autoclass: barrel.train.components.normalization.normalizers.DatasetStatsNormalizer
|
| 435 |
-
component_name: translation
|
| 436 |
-
mode: bounds_q99
|
| 437 |
-
stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
|
| 438 |
-
stats_key: ''
|
| 439 |
-
vlm_config:
|
| 440 |
-
attn_implementation: flash_attention_2
|
| 441 |
-
autoclass: barrel.train.pipes.vlams.models.vlms.paligemma.PaliGemmaVLM
|
| 442 |
-
depth_tokens: 0
|
| 443 |
-
lm_head: false
|
| 444 |
-
mean_resizing: false
|
| 445 |
-
model_id: google/paligemma-3b-mix-224
|
| 446 |
-
paligemma_3d_config: {}
|
| 447 |
-
pretrain_config:
|
| 448 |
-
exclude_modules: []
|
| 449 |
-
fine_tuneable_modules: []
|
| 450 |
-
frozen_dtype: bfloat16
|
| 451 |
-
state_dict:
|
| 452 |
-
checkpoint_path: PRETRAINED_MODULE
|
| 453 |
-
drop_keys: []
|
| 454 |
-
hf_filename: null
|
| 455 |
-
hf_hub_repo: null
|
| 456 |
-
load_key: ''
|
| 457 |
-
remap_keys: {}
|
| 458 |
-
trainable: true
|
| 459 |
-
processor_config:
|
| 460 |
-
autoclass: barrel.train.pipes.vlams.types.vlm_processors.PaliGemmaProcessor
|
| 461 |
-
image_sizes:
|
| 462 |
-
main:
|
| 463 |
-
height: 224
|
| 464 |
-
width: 224
|
| 465 |
-
image_token: <image>
|
| 466 |
-
max_language_tokens: -1
|
| 467 |
-
train_only_depth_tokens: false
|
| 468 |
-
optimizer_config:
|
| 469 |
-
clip_grad_norm: 1.0
|
| 470 |
-
grad_scaler:
|
| 471 |
-
enabled: false
|
| 472 |
-
optimizer:
|
| 473 |
-
autoclass: barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer
|
| 474 |
-
hyperparams:
|
| 475 |
-
lr: 5.0e-05
|
| 476 |
-
weight_decay: 0.0
|
| 477 |
-
param_groups: {}
|
| 478 |
-
torch_optimizer: torch.optim.AdamW
|
| 479 |
-
scheduler:
|
| 480 |
-
autoclass: barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler
|
| 481 |
-
milestones:
|
| 482 |
-
- 500
|
| 483 |
-
schedulers:
|
| 484 |
-
warmup:
|
| 485 |
-
autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
|
| 486 |
-
hyperparams:
|
| 487 |
-
start_factor: 0.0002
|
| 488 |
-
end_factor: 1.0
|
| 489 |
-
total_iters: 500
|
| 490 |
-
torch_scheduler: torch.optim.lr_scheduler.LinearLR
|
| 491 |
-
const:
|
| 492 |
-
autoclass: barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler
|
| 493 |
-
hyperparams:
|
| 494 |
-
factor: 1.0
|
| 495 |
-
total_iters: 300000
|
| 496 |
-
torch_scheduler: torch.optim.lr_scheduler.ConstantLR
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bd5e0b76cb6b7291610aafbeb6924c2760b7ecb4640f84ecb78b3762fe7872be
|
| 3 |
+
size 16020
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/format.log
CHANGED
|
@@ -1,24 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/modeling_pizero_fm_paligemma.py
|
| 7 |
-
Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/configuration_pizero_fm_paligemma.py
|
| 8 |
-
Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/common_pizero_fm_paligemma.py
|
| 9 |
-
Fixing /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/processing_pizero_fm_paligemma.py
|
| 10 |
-
|
| 11 |
-
reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/configuration_pizero_fm_paligemma.py
|
| 12 |
-
reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/common_pizero_fm_paligemma.py
|
| 13 |
-
reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/processing_pizero_fm_paligemma.py
|
| 14 |
-
reformatted /home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/modeling_pizero_fm_paligemma.py
|
| 15 |
-
|
| 16 |
-
All done! ✨ 🍰 ✨
|
| 17 |
-
4 files reformatted.
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
TO RERUN:
|
| 21 |
-
|
| 22 |
-
bazel run //tools:autoflake -- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m autoflake
|
| 23 |
-
bazel run //tools:isort -- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m isort
|
| 24 |
-
bazel run //tools:black -- /scratch/petko_petkov/.cache/bazel/_bazel_petko_petkov/e534278fe244a0db19bf3b7c5ecfa723/execroot/_main/bazel-out/k8-opt-ST-4fb24e704967/bin/barrel/train/pipes/vlams/train.runfiles/_main/barrel/train/pipes/vlams/_train.venv/bin/python3 -m black
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fe0e32cf3f9159943e77c44dc0bae6d89c832b00493874572a1402e909ee9e38
|
| 3 |
+
size 5564
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/hf_export/gargantuan-green-herring/src/model_config.yaml
CHANGED
|
@@ -1,120 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
activation: GELU
|
| 5 |
-
activation_kwargs:
|
| 6 |
-
approximate: tanh
|
| 7 |
-
attn_implementation: sdpa
|
| 8 |
-
dropout: 0.0
|
| 9 |
-
feature_size: 1024
|
| 10 |
-
head_dim: 256
|
| 11 |
-
hidden_size: 4096
|
| 12 |
-
norm: RMSNorm
|
| 13 |
-
num_heads: 8
|
| 14 |
-
num_kv_heads: 1
|
| 15 |
-
position_embed_config:
|
| 16 |
-
base: 10000
|
| 17 |
-
cached: true
|
| 18 |
-
embedding_dim: 256
|
| 19 |
-
num_embeddings: 512
|
| 20 |
-
num_blocks: 18
|
| 21 |
-
noised_control_proj_config:
|
| 22 |
-
activation: SiLU
|
| 23 |
-
layers:
|
| 24 |
-
- 8
|
| 25 |
-
- 2048
|
| 26 |
-
- 1024
|
| 27 |
-
- 1024
|
| 28 |
-
norm: null
|
| 29 |
-
time_embed:
|
| 30 |
-
activation: SiLU
|
| 31 |
-
layers: []
|
| 32 |
-
learnable_features: false
|
| 33 |
-
max_period: 10000.0
|
| 34 |
-
norm: null
|
| 35 |
-
num_features: 1024
|
| 36 |
-
robot_state_proj_config:
|
| 37 |
-
activation: SiLU
|
| 38 |
-
fourier: false
|
| 39 |
-
layers:
|
| 40 |
-
- 8
|
| 41 |
-
- 1024
|
| 42 |
-
mode: ee_pose_gripper
|
| 43 |
-
rotation_components: 4
|
| 44 |
-
token_size: 1024
|
| 45 |
-
processor_config:
|
| 46 |
-
control_io_config:
|
| 47 |
-
future_control_offset_sec: 0.0
|
| 48 |
-
future_controls_sequence_length: 5
|
| 49 |
-
future_controls_sequence_stride_sec: 0.2
|
| 50 |
-
future_frames_sequence_length: 1
|
| 51 |
-
future_frames_sequence_stride_sec: null
|
| 52 |
-
past_frames_sequence_length: 1
|
| 53 |
-
past_frames_stride_sec: null
|
| 54 |
-
past_scalars_sequence_length: 1
|
| 55 |
-
past_scalars_stride_sec: null
|
| 56 |
-
sequence_frames: 1
|
| 57 |
-
sequence_frames_stride_sec: null
|
| 58 |
-
control_tokenizer_config: {}
|
| 59 |
-
distribution_hyperparams:
|
| 60 |
-
alpha: 1.5
|
| 61 |
-
beta: 1.0
|
| 62 |
-
image_resize: naive
|
| 63 |
-
joints_obs_norm:
|
| 64 |
-
high:
|
| 65 |
-
- 3.141592653589793
|
| 66 |
-
- 3.141592653589793
|
| 67 |
-
- 3.141592653589793
|
| 68 |
-
- 3.141592653589793
|
| 69 |
-
- 3.141592653589793
|
| 70 |
-
- 3.141592653589793
|
| 71 |
-
- 3.141592653589793
|
| 72 |
-
low:
|
| 73 |
-
- -3.141592653589793
|
| 74 |
-
- -3.141592653589793
|
| 75 |
-
- -3.141592653589793
|
| 76 |
-
- -3.141592653589793
|
| 77 |
-
- -3.141592653589793
|
| 78 |
-
- -3.141592653589793
|
| 79 |
-
- -3.141592653589793
|
| 80 |
-
num_inference_steps: 10
|
| 81 |
-
r0_distribution: uniform
|
| 82 |
-
rotation_control_frame: eef_delta
|
| 83 |
-
rotation_control_norm:
|
| 84 |
-
factor: 4.0
|
| 85 |
-
rotation_format: quaternion
|
| 86 |
-
rotation_obs_frame: robot_base
|
| 87 |
-
rotation_obs_norm: {}
|
| 88 |
-
sig_min: 0.001
|
| 89 |
-
timestep_distribution: beta
|
| 90 |
-
translation_control_frame: robot_base_delta
|
| 91 |
-
translation_control_norm:
|
| 92 |
-
high:
|
| 93 |
-
- 0.04
|
| 94 |
-
- 0.04
|
| 95 |
-
- 0.04
|
| 96 |
-
low:
|
| 97 |
-
- -0.04
|
| 98 |
-
- -0.04
|
| 99 |
-
- -0.04
|
| 100 |
-
translation_obs_frame: robot_base
|
| 101 |
-
translation_obs_norm:
|
| 102 |
-
component_name: translation
|
| 103 |
-
mode: bounds_q99
|
| 104 |
-
stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
|
| 105 |
-
stats_key: ''
|
| 106 |
-
vlm_config:
|
| 107 |
-
attn_implementation: sdpa
|
| 108 |
-
depth_tokens: 0
|
| 109 |
-
lm_head: false
|
| 110 |
-
mean_resizing: false
|
| 111 |
-
model_id: google/paligemma-3b-mix-224
|
| 112 |
-
paligemma_3d_config: {}
|
| 113 |
-
processor_config:
|
| 114 |
-
image_sizes:
|
| 115 |
-
main:
|
| 116 |
-
height: 224
|
| 117 |
-
width: 224
|
| 118 |
-
image_token: <image>
|
| 119 |
-
max_language_tokens: 64
|
| 120 |
-
train_only_depth_tokens: false
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4f892712e82673a0b76048f0d0fb4a492813c36df8ecbcc54265c66d697dac86
|
| 3 |
+
size 2819
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/language_results.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fa48b47d0eb7f4fa77de8d2ab1bb37f2b726e54e20862f9ee6e95e2be09c7745
|
| 3 |
+
size 544466
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/model_config.yaml
CHANGED
|
@@ -1,137 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
control_decoder_config:
|
| 5 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoder
|
| 6 |
-
block_config:
|
| 7 |
-
activation: GELU
|
| 8 |
-
activation_kwargs:
|
| 9 |
-
approximate: tanh
|
| 10 |
-
attn_implementation: sdpa
|
| 11 |
-
autoclass: barrel.train.pipes.vlams.models.control.pizero_fm_decoder.PiZeroFlowMatchingDecoderBlock
|
| 12 |
-
dropout: 0.0
|
| 13 |
-
feature_size: 1024
|
| 14 |
-
head_dim: 256
|
| 15 |
-
hidden_size: 4096
|
| 16 |
-
norm: RMSNorm
|
| 17 |
-
num_heads: 8
|
| 18 |
-
num_kv_heads: 1
|
| 19 |
-
position_embed_config:
|
| 20 |
-
autoclass: barrel.train.components.nn.layers.positional_encodings.RotaryPositionalEncoding
|
| 21 |
-
base: 10000
|
| 22 |
-
cached: true
|
| 23 |
-
embedding_dim: 256
|
| 24 |
-
num_embeddings: 512
|
| 25 |
-
num_blocks: 18
|
| 26 |
-
noised_control_proj_config:
|
| 27 |
-
activation: SiLU
|
| 28 |
-
autoclass: barrel.train.pipes.vlams.models.control.token_proj.NoisedControlProjector
|
| 29 |
-
layers:
|
| 30 |
-
- 8
|
| 31 |
-
- 2048
|
| 32 |
-
- 1024
|
| 33 |
-
- 1024
|
| 34 |
-
norm: null
|
| 35 |
-
time_embed:
|
| 36 |
-
activation: SiLU
|
| 37 |
-
autoclass: barrel.train.components.nn.layers.fourier_features.FourierFeatures
|
| 38 |
-
layers: []
|
| 39 |
-
learnable_features: false
|
| 40 |
-
max_period: 10000.0
|
| 41 |
-
norm: null
|
| 42 |
-
num_features: 1024
|
| 43 |
-
robot_state_proj_config:
|
| 44 |
-
activation: SiLU
|
| 45 |
-
autoclass: barrel.train.pipes.vlams.models.control.token_proj.RobotStateProjector
|
| 46 |
-
fourier: false
|
| 47 |
-
layers:
|
| 48 |
-
- 8
|
| 49 |
-
- 1024
|
| 50 |
-
mode: ee_pose_gripper
|
| 51 |
-
rotation_components: 4
|
| 52 |
-
token_size: 1024
|
| 53 |
-
processor_config:
|
| 54 |
-
autoclass: barrel.train.pipes.vlams.types.vlam_processors.PiZeroFlowMatchingProcessor
|
| 55 |
-
control_io_config:
|
| 56 |
-
future_control_offset_sec: 0.0
|
| 57 |
-
future_controls_sequence_length: 5
|
| 58 |
-
future_controls_sequence_stride_sec: 0.2
|
| 59 |
-
future_frames_sequence_length: 1
|
| 60 |
-
future_frames_sequence_stride_sec: null
|
| 61 |
-
past_frames_sequence_length: 1
|
| 62 |
-
past_frames_stride_sec: null
|
| 63 |
-
past_scalars_sequence_length: 1
|
| 64 |
-
past_scalars_stride_sec: null
|
| 65 |
-
sequence_frames: 1
|
| 66 |
-
sequence_frames_stride_sec: null
|
| 67 |
-
control_tokenizer_config:
|
| 68 |
-
autoclass: barrel.train.pipes.vlams.types.action_tokenizer.EmptyTokenizer
|
| 69 |
-
distribution_hyperparams:
|
| 70 |
-
alpha: 1.5
|
| 71 |
-
beta: 1.0
|
| 72 |
-
image_resize: naive
|
| 73 |
-
joints_obs_norm:
|
| 74 |
-
autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
|
| 75 |
-
high:
|
| 76 |
-
- 3.141592653589793
|
| 77 |
-
- 3.141592653589793
|
| 78 |
-
- 3.141592653589793
|
| 79 |
-
- 3.141592653589793
|
| 80 |
-
- 3.141592653589793
|
| 81 |
-
- 3.141592653589793
|
| 82 |
-
- 3.141592653589793
|
| 83 |
-
low:
|
| 84 |
-
- -3.141592653589793
|
| 85 |
-
- -3.141592653589793
|
| 86 |
-
- -3.141592653589793
|
| 87 |
-
- -3.141592653589793
|
| 88 |
-
- -3.141592653589793
|
| 89 |
-
- -3.141592653589793
|
| 90 |
-
- -3.141592653589793
|
| 91 |
-
num_inference_steps: 10
|
| 92 |
-
r0_distribution: uniform
|
| 93 |
-
rotation_control_frame: eef_delta
|
| 94 |
-
rotation_control_norm:
|
| 95 |
-
autoclass: barrel.train.components.normalization.normalizers.RotationStereomapNormalizer
|
| 96 |
-
factor: 4.0
|
| 97 |
-
rotation_format: quaternion
|
| 98 |
-
rotation_obs_frame: robot_base
|
| 99 |
-
rotation_obs_norm:
|
| 100 |
-
autoclass: barrel.train.components.normalization.normalizers.IdentityNormalizer
|
| 101 |
-
sig_min: 0.001
|
| 102 |
-
timestep_distribution: beta
|
| 103 |
-
translation_control_frame: robot_base_delta
|
| 104 |
-
translation_control_norm:
|
| 105 |
-
autoclass: barrel.train.components.normalization.normalizers.BoundsNormalizer
|
| 106 |
-
high:
|
| 107 |
-
- 0.04
|
| 108 |
-
- 0.04
|
| 109 |
-
- 0.04
|
| 110 |
-
low:
|
| 111 |
-
- -0.04
|
| 112 |
-
- -0.04
|
| 113 |
-
- -0.04
|
| 114 |
-
translation_obs_frame: robot_base
|
| 115 |
-
translation_obs_norm:
|
| 116 |
-
autoclass: barrel.train.components.normalization.normalizers.DatasetStatsNormalizer
|
| 117 |
-
component_name: translation
|
| 118 |
-
mode: bounds_q99
|
| 119 |
-
stats_filepath: barrel/train/pipes/vlams/types/observation_stats.yaml
|
| 120 |
-
stats_key: ''
|
| 121 |
-
vlm_config:
|
| 122 |
-
attn_implementation: flash_attention_2
|
| 123 |
-
autoclass: barrel.train.pipes.vlams.models.vlms.paligemma.PaliGemmaVLM
|
| 124 |
-
depth_tokens: 0
|
| 125 |
-
lm_head: false
|
| 126 |
-
mean_resizing: false
|
| 127 |
-
model_id: google/paligemma-3b-mix-224
|
| 128 |
-
paligemma_3d_config: {}
|
| 129 |
-
processor_config:
|
| 130 |
-
autoclass: barrel.train.pipes.vlams.types.vlm_processors.PaliGemmaProcessor
|
| 131 |
-
image_sizes:
|
| 132 |
-
main:
|
| 133 |
-
height: 224
|
| 134 |
-
width: 224
|
| 135 |
-
image_token: <image>
|
| 136 |
-
max_language_tokens: -1
|
| 137 |
-
train_only_depth_tokens: false
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bfb3f9546044efb942cefc95d211e03d5cda1aeb8d25bd9f29a7fe69ff5bb792
|
| 3 |
+
size 4278
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/raw_config.yaml
CHANGED
|
@@ -1,11 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
tag: bridge_curriculum_phase1
|
| 5 |
-
|
| 6 |
-
num_steps: &num_steps 15000
|
| 7 |
-
|
| 8 |
-
callbacks_config:
|
| 9 |
-
callbacks:
|
| 10 |
-
checkpoint_callback:
|
| 11 |
-
keep_model_steps: !overwrite [15000]
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a0b90ff29d4b11cf45658aa01d93964b33108ab0eb7c5915001e05f896ff1435
|
| 3 |
+
size 218
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session.log
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f24870cf60647678b0e12eb39f8c8ef37eab3922c28235fff5497dbe23ad6e5d
|
| 3 |
+
size 18986089
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_info.json
CHANGED
|
@@ -1 +1,3 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:df0ab680e7e76361f4cd0df6923b4fb35a011220a468dd1ba1a01d83205e7c1b
|
| 3 |
+
size 936
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_1.log
CHANGED
|
@@ -1,526 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
lr=5e-05,
|
| 5 |
-
weight_decay=0.0,
|
| 6 |
-
),
|
| 7 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 8 |
-
torch_optimizer='torch.optim.AdamW',
|
| 9 |
-
), scheduler=SequentialSchedulerConfig(
|
| 10 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 11 |
-
milestones=ConfigList[int](
|
| 12 |
-
500,
|
| 13 |
-
),
|
| 14 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 15 |
-
warmup=TorchSchedulerConfig(
|
| 16 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 17 |
-
hyperparams=ConfigDict[typing.Any](
|
| 18 |
-
start_factor=0.0002,
|
| 19 |
-
end_factor=1.0,
|
| 20 |
-
total_iters=500,
|
| 21 |
-
),
|
| 22 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 23 |
-
),
|
| 24 |
-
const=TorchSchedulerConfig(
|
| 25 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 26 |
-
hyperparams=ConfigDict[typing.Any](
|
| 27 |
-
factor=1.0,
|
| 28 |
-
total_iters=300000,
|
| 29 |
-
),
|
| 30 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 31 |
-
),
|
| 32 |
-
),
|
| 33 |
-
)
|
| 34 |
-
INFO 2026-04-05 07:47:39,914 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 35 |
-
INFO 2026-04-05 07:47:41,966 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge
|
| 36 |
-
INFO 2026-04-05 07:48:06,548 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 37 |
-
INFO 2026-04-05 07:48:10,913 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 38 |
-
INFO 2026-04-05 07:48:10,913 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=28947.0, dataset_name=bridge
|
| 39 |
-
INFO 2026-04-05 07:48:10,953 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183254, rank=1
|
| 40 |
-
INFO 2026-04-05 07:48:11,805 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge rank=1, status=start
|
| 41 |
-
INFO 2026-04-05 07:48:11,844 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
|
| 42 |
-
INFO 2026-04-05 07:48:11,846 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=31931.7
|
| 43 |
-
INFO 2026-04-05 07:48:34,458 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=1
|
| 44 |
-
INFO 2026-04-05 07:48:34,460 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 45 |
-
INFO 2026-04-05 07:48:34,519 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-2, worker_pid=3875752, parent_pid=3872890, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 46 |
-
INFO 2026-04-05 07:48:34,552 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-2, worker_pid=3875770, parent_pid=3872890, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 47 |
-
INFO 2026-04-05 07:51:52,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 48 |
-
INFO 2026-04-05 07:54:44,997 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 49 |
-
INFO 2026-04-05 07:57:35,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 50 |
-
INFO 2026-04-05 08:00:26,862 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 51 |
-
INFO 2026-04-05 08:03:18,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 52 |
-
INFO 2026-04-05 08:06:09,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 53 |
-
INFO 2026-04-05 08:09:01,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 54 |
-
INFO 2026-04-05 08:11:53,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 55 |
-
INFO 2026-04-05 08:14:44,578 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 56 |
-
INFO 2026-04-05 08:17:37,375 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 57 |
-
INFO 2026-04-05 08:20:30,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 58 |
-
INFO 2026-04-05 08:23:23,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 59 |
-
INFO 2026-04-05 08:26:15,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 60 |
-
INFO 2026-04-05 08:29:07,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 61 |
-
INFO 2026-04-05 08:31:59,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 62 |
-
INFO 2026-04-05 08:34:50,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 63 |
-
INFO 2026-04-05 08:37:43,456 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 64 |
-
INFO 2026-04-05 08:40:35,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 65 |
-
INFO 2026-04-05 08:43:28,467 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 66 |
-
INFO 2026-04-05 08:46:21,132 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 67 |
-
INFO 2026-04-05 08:49:13,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 68 |
-
INFO 2026-04-05 08:52:04,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 69 |
-
INFO 2026-04-05 08:54:56,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 70 |
-
INFO 2026-04-05 08:57:48,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 71 |
-
INFO 2026-04-05 09:00:40,238 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 72 |
-
INFO 2026-04-05 09:03:32,325 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 73 |
-
INFO 2026-04-05 09:06:23,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 74 |
-
INFO 2026-04-05 09:09:15,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 75 |
-
INFO 2026-04-05 09:12:08,117 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 76 |
-
INFO 2026-04-05 09:14:59,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 77 |
-
INFO 2026-04-05 09:17:51,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 78 |
-
INFO 2026-04-05 09:20:42,298 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 79 |
-
INFO 2026-04-05 09:23:34,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 80 |
-
INFO 2026-04-05 09:26:26,064 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 81 |
-
INFO 2026-04-05 09:29:18,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 82 |
-
INFO 2026-04-05 09:32:09,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 83 |
-
INFO 2026-04-05 09:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 84 |
-
INFO 2026-04-05 09:37:52,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 85 |
-
INFO 2026-04-05 09:40:43,428 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 86 |
-
INFO 2026-04-05 09:43:34,586 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 87 |
-
INFO 2026-04-05 09:46:26,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 88 |
-
INFO 2026-04-05 09:49:17,267 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 89 |
-
INFO 2026-04-05 09:52:06,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 90 |
-
INFO 2026-04-05 09:54:57,889 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 91 |
-
INFO 2026-04-05 09:57:48,894 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 92 |
-
INFO 2026-04-05 10:00:41,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 93 |
-
INFO 2026-04-05 10:03:33,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 94 |
-
INFO 2026-04-05 10:06:24,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 95 |
-
INFO 2026-04-05 10:09:17,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 96 |
-
INFO 2026-04-05 10:12:18,772 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 97 |
-
INFO 2026-04-05 10:15:33,483 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 98 |
-
INFO 2026-04-05 10:18:24,744 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 99 |
-
INFO 2026-04-05 10:21:17,303 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 100 |
-
INFO 2026-04-05 10:24:08,917 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 101 |
-
INFO 2026-04-05 10:27:00,005 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 102 |
-
INFO 2026-04-05 10:29:51,959 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 103 |
-
INFO 2026-04-05 10:32:42,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 104 |
-
INFO 2026-04-05 10:35:34,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 105 |
-
INFO 2026-04-05 10:38:25,583 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 106 |
-
INFO 2026-04-05 10:41:18,350 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 107 |
-
INFO 2026-04-05 10:44:08,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 108 |
-
INFO 2026-04-05 10:47:00,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 109 |
-
INFO 2026-04-05 10:49:51,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 110 |
-
INFO 2026-04-05 10:52:43,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 111 |
-
INFO 2026-04-05 10:55:35,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 112 |
-
INFO 2026-04-05 10:58:26,949 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 113 |
-
INFO 2026-04-05 11:01:18,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 114 |
-
INFO 2026-04-05 11:04:10,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 115 |
-
INFO 2026-04-05 11:07:02,461 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 116 |
-
INFO 2026-04-05 11:09:53,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 117 |
-
INFO 2026-04-05 11:12:45,451 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 118 |
-
INFO 2026-04-05 11:15:36,492 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 119 |
-
INFO 2026-04-05 11:18:28,400 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 120 |
-
INFO 2026-04-05 11:21:20,007 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 121 |
-
INFO 2026-04-05 11:24:11,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 122 |
-
INFO 2026-04-05 11:27:02,198 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 123 |
-
INFO 2026-04-05 11:29:53,736 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 124 |
-
INFO 2026-04-05 11:32:44,011 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 125 |
-
INFO 2026-04-05 11:35:35,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 126 |
-
INFO 2026-04-05 11:38:25,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 127 |
-
INFO 2026-04-05 11:41:15,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 128 |
-
INFO 2026-04-05 11:44:05,134 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 129 |
-
INFO 2026-04-05 11:46:56,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 130 |
-
INFO 2026-04-05 11:49:47,469 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 131 |
-
INFO 2026-04-05 11:52:38,563 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 132 |
-
INFO 2026-04-05 11:55:30,290 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 133 |
-
INFO 2026-04-05 11:58:21,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 134 |
-
INFO 2026-04-05 12:01:13,424 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 135 |
-
INFO 2026-04-05 12:04:04,015 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 136 |
-
INFO 2026-04-05 12:06:55,196 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 137 |
-
INFO 2026-04-05 12:09:46,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 138 |
-
INFO 2026-04-05 12:12:37,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 139 |
-
INFO 2026-04-05 12:15:28,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 140 |
-
INFO 2026-04-05 12:18:19,903 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 141 |
-
INFO 2026-04-05 12:21:11,565 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 142 |
-
INFO 2026-04-05 12:24:02,399 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 143 |
-
INFO 2026-04-05 12:26:54,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 144 |
-
INFO 2026-04-05 12:29:45,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 145 |
-
INFO 2026-04-05 12:32:36,286 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 146 |
-
INFO 2026-04-05 12:35:36,863 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 147 |
-
INFO 2026-04-05 12:38:50,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 148 |
-
INFO 2026-04-05 12:41:41,343 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 149 |
-
INFO 2026-04-05 12:44:32,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 150 |
-
INFO 2026-04-05 12:47:23,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 151 |
-
INFO 2026-04-05 12:50:13,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 152 |
-
INFO 2026-04-05 12:53:06,274 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 153 |
-
INFO 2026-04-05 12:55:56,980 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 154 |
-
INFO 2026-04-05 12:58:48,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 155 |
-
INFO 2026-04-05 13:01:39,887 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 156 |
-
INFO 2026-04-05 13:04:31,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 157 |
-
INFO 2026-04-05 13:07:22,319 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 158 |
-
INFO 2026-04-05 13:10:15,398 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 159 |
-
INFO 2026-04-05 13:13:06,519 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 160 |
-
INFO 2026-04-05 13:15:57,164 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 161 |
-
INFO 2026-04-05 13:18:49,647 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 162 |
-
INFO 2026-04-05 13:21:40,162 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 163 |
-
INFO 2026-04-05 13:24:31,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 164 |
-
INFO 2026-04-05 13:27:22,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 165 |
-
INFO 2026-04-05 13:30:13,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 166 |
-
INFO 2026-04-05 13:33:06,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 167 |
-
INFO 2026-04-05 13:35:57,606 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 168 |
-
INFO 2026-04-05 13:38:47,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 169 |
-
INFO 2026-04-05 13:41:40,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 170 |
-
INFO 2026-04-05 13:44:31,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 171 |
-
INFO 2026-04-05 13:47:23,254 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 172 |
-
INFO 2026-04-05 13:50:14,660 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 173 |
-
INFO 2026-04-05 13:53:06,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 174 |
-
INFO 2026-04-05 13:55:58,107 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 175 |
-
INFO 2026-04-05 13:58:50,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 176 |
-
INFO 2026-04-05 14:01:41,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 177 |
-
INFO 2026-04-05 14:04:32,278 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 178 |
-
INFO 2026-04-05 14:07:24,300 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 179 |
-
INFO 2026-04-05 14:10:15,785 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 180 |
-
INFO 2026-04-05 14:13:07,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 181 |
-
INFO 2026-04-05 14:15:58,762 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 182 |
-
INFO 2026-04-05 14:18:49,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 183 |
-
INFO 2026-04-05 14:21:40,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 184 |
-
INFO 2026-04-05 14:24:31,804 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 185 |
-
INFO 2026-04-05 14:27:22,607 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 186 |
-
INFO 2026-04-05 14:30:14,379 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 187 |
-
INFO 2026-04-05 14:33:06,049 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 188 |
-
INFO 2026-04-05 14:35:56,094 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 189 |
-
INFO 2026-04-05 14:38:48,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 190 |
-
INFO 2026-04-05 14:41:40,352 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 191 |
-
INFO 2026-04-05 14:44:32,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 192 |
-
INFO 2026-04-05 14:47:25,215 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 193 |
-
INFO 2026-04-05 14:50:15,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 194 |
-
INFO 2026-04-05 14:53:07,347 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 195 |
-
INFO 2026-04-05 14:56:00,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 196 |
-
INFO 2026-04-05 14:59:02,313 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 197 |
-
INFO 2026-04-05 14:59:35,376 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=1
|
| 198 |
-
INFO 2026-04-06 07:33:53,247 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=1, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 199 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 200 |
-
hyperparams=ConfigDict[typing.Any](
|
| 201 |
-
lr=5e-05,
|
| 202 |
-
weight_decay=0.0,
|
| 203 |
-
),
|
| 204 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 205 |
-
torch_optimizer='torch.optim.AdamW',
|
| 206 |
-
), scheduler=SequentialSchedulerConfig(
|
| 207 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 208 |
-
milestones=ConfigList[int](
|
| 209 |
-
500,
|
| 210 |
-
),
|
| 211 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 212 |
-
warmup=TorchSchedulerConfig(
|
| 213 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 214 |
-
hyperparams=ConfigDict[typing.Any](
|
| 215 |
-
start_factor=0.0002,
|
| 216 |
-
end_factor=1.0,
|
| 217 |
-
total_iters=500,
|
| 218 |
-
),
|
| 219 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 220 |
-
),
|
| 221 |
-
const=TorchSchedulerConfig(
|
| 222 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 223 |
-
hyperparams=ConfigDict[typing.Any](
|
| 224 |
-
factor=1.0,
|
| 225 |
-
total_iters=300000,
|
| 226 |
-
),
|
| 227 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 228 |
-
),
|
| 229 |
-
),
|
| 230 |
-
)
|
| 231 |
-
INFO 2026-04-06 07:33:53,250 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 232 |
-
INFO 2026-04-06 07:33:56,251 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 233 |
-
INFO 2026-04-06 07:34:13,988 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 234 |
-
INFO 2026-04-06 07:34:19,398 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 235 |
-
INFO 2026-04-06 07:34:23,946 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 236 |
-
INFO 2026-04-06 07:34:23,948 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=27697.1, dataset_name=bridge_paraphrase_k5
|
| 237 |
-
INFO 2026-04-06 07:34:23,993 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183254, rank=1
|
| 238 |
-
INFO 2026-04-06 07:34:24,858 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=1, status=start
|
| 239 |
-
INFO 2026-04-06 07:34:24,906 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
|
| 240 |
-
INFO 2026-04-06 07:34:24,910 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=31660.3
|
| 241 |
-
INFO 2026-04-06 07:34:24,918 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=1, status=start
|
| 242 |
-
INFO 2026-04-06 07:36:05,815 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=1, status=success
|
| 243 |
-
INFO 2026-04-06 07:36:27,875 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=1
|
| 244 |
-
INFO 2026-04-06 07:36:27,879 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 245 |
-
INFO 2026-04-06 07:36:27,937 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=2465169, parent_pid=2461886, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 246 |
-
INFO 2026-04-06 07:36:27,977 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=2465195, parent_pid=2461886, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 247 |
-
INFO 2026-04-06 07:39:43,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 248 |
-
INFO 2026-04-06 07:42:35,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 249 |
-
INFO 2026-04-06 07:45:26,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 250 |
-
INFO 2026-04-06 07:48:18,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 251 |
-
INFO 2026-04-06 07:51:08,587 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 252 |
-
INFO 2026-04-06 07:54:00,463 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 253 |
-
INFO 2026-04-06 07:56:51,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 254 |
-
INFO 2026-04-06 07:59:43,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 255 |
-
INFO 2026-04-06 08:02:34,656 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 256 |
-
INFO 2026-04-06 08:05:26,547 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 257 |
-
INFO 2026-04-06 08:08:18,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 258 |
-
INFO 2026-04-06 08:11:11,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 259 |
-
INFO 2026-04-06 08:14:03,148 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 260 |
-
INFO 2026-04-06 08:16:57,119 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 261 |
-
INFO 2026-04-06 08:19:48,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 262 |
-
INFO 2026-04-06 08:22:39,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 263 |
-
INFO 2026-04-06 08:25:31,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 264 |
-
INFO 2026-04-06 08:28:25,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 265 |
-
INFO 2026-04-06 08:31:16,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 266 |
-
INFO 2026-04-06 08:34:09,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 267 |
-
INFO 2026-04-06 08:37:01,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 268 |
-
INFO 2026-04-06 08:39:53,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 269 |
-
INFO 2026-04-06 08:42:45,381 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 270 |
-
INFO 2026-04-06 08:45:38,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 271 |
-
INFO 2026-04-06 08:48:30,989 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 272 |
-
INFO 2026-04-06 08:51:23,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 273 |
-
INFO 2026-04-06 08:54:13,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 274 |
-
INFO 2026-04-06 08:57:05,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 275 |
-
INFO 2026-04-06 08:59:57,948 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 276 |
-
INFO 2026-04-06 09:02:49,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 277 |
-
INFO 2026-04-06 09:05:41,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 278 |
-
INFO 2026-04-06 09:08:33,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 279 |
-
INFO 2026-04-06 09:11:25,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 280 |
-
INFO 2026-04-06 09:14:17,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 281 |
-
INFO 2026-04-06 09:17:08,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 282 |
-
INFO 2026-04-06 09:20:01,791 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 283 |
-
INFO 2026-04-06 09:22:53,270 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 284 |
-
INFO 2026-04-06 09:25:46,358 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 285 |
-
INFO 2026-04-06 09:28:40,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 286 |
-
INFO 2026-04-06 09:31:33,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 287 |
-
INFO 2026-04-06 09:34:24,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 288 |
-
INFO 2026-04-06 09:37:17,501 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 289 |
-
INFO 2026-04-06 09:40:08,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 290 |
-
INFO 2026-04-06 09:43:00,699 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 291 |
-
INFO 2026-04-06 09:45:52,659 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 292 |
-
INFO 2026-04-06 09:48:44,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 293 |
-
INFO 2026-04-06 09:51:36,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 294 |
-
INFO 2026-04-06 09:54:29,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 295 |
-
INFO 2026-04-06 09:57:20,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 296 |
-
INFO 2026-04-06 10:00:22,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 297 |
-
INFO 2026-04-06 10:03:48,882 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 298 |
-
INFO 2026-04-06 10:06:52,075 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 299 |
-
INFO 2026-04-06 10:09:59,025 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 300 |
-
INFO 2026-04-06 10:13:05,837 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 301 |
-
INFO 2026-04-06 10:16:11,930 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 302 |
-
INFO 2026-04-06 10:19:15,973 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 303 |
-
INFO 2026-04-06 10:22:24,656 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 304 |
-
INFO 2026-04-06 10:25:33,535 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 305 |
-
INFO 2026-04-06 10:28:42,180 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 306 |
-
INFO 2026-04-06 10:31:51,507 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 307 |
-
INFO 2026-04-06 10:35:00,437 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 308 |
-
INFO 2026-04-06 10:38:08,598 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 309 |
-
INFO 2026-04-06 10:41:15,590 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 310 |
-
INFO 2026-04-06 11:29:31,531 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=1, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 311 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 312 |
-
hyperparams=ConfigDict[typing.Any](
|
| 313 |
-
lr=5e-05,
|
| 314 |
-
weight_decay=0.0,
|
| 315 |
-
),
|
| 316 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 317 |
-
torch_optimizer='torch.optim.AdamW',
|
| 318 |
-
), scheduler=SequentialSchedulerConfig(
|
| 319 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 320 |
-
milestones=ConfigList[int](
|
| 321 |
-
500,
|
| 322 |
-
),
|
| 323 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 324 |
-
warmup=TorchSchedulerConfig(
|
| 325 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 326 |
-
hyperparams=ConfigDict[typing.Any](
|
| 327 |
-
start_factor=0.0002,
|
| 328 |
-
end_factor=1.0,
|
| 329 |
-
total_iters=500,
|
| 330 |
-
),
|
| 331 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 332 |
-
),
|
| 333 |
-
const=TorchSchedulerConfig(
|
| 334 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 335 |
-
hyperparams=ConfigDict[typing.Any](
|
| 336 |
-
factor=1.0,
|
| 337 |
-
total_iters=300000,
|
| 338 |
-
),
|
| 339 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 340 |
-
),
|
| 341 |
-
),
|
| 342 |
-
)
|
| 343 |
-
INFO 2026-04-06 11:29:31,533 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 344 |
-
INFO 2026-04-06 11:29:34,907 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 345 |
-
INFO 2026-04-06 11:29:43,556 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 346 |
-
INFO 2026-04-06 11:29:47,858 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 347 |
-
INFO 2026-04-06 11:29:51,526 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 348 |
-
INFO 2026-04-06 11:29:51,527 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=16619.9, dataset_name=bridge_paraphrase_k5
|
| 349 |
-
INFO 2026-04-06 11:29:51,568 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183254, rank=1
|
| 350 |
-
INFO 2026-04-06 11:29:52,392 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=1, status=start
|
| 351 |
-
INFO 2026-04-06 11:29:52,431 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
|
| 352 |
-
INFO 2026-04-06 11:29:52,433 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=20899.3
|
| 353 |
-
INFO 2026-04-06 11:29:52,439 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=1, status=start
|
| 354 |
-
INFO 2026-04-06 11:31:39,839 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=1, status=success
|
| 355 |
-
INFO 2026-04-06 11:32:01,842 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=1
|
| 356 |
-
INFO 2026-04-06 11:32:01,844 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 357 |
-
INFO 2026-04-06 11:32:02,009 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-7, worker_pid=3492780, parent_pid=3489441, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 358 |
-
INFO 2026-04-06 11:32:02,044 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-7, worker_pid=3492815, parent_pid=3489441, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 359 |
-
INFO 2026-04-06 11:35:29,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 360 |
-
INFO 2026-04-06 11:38:35,205 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 361 |
-
INFO 2026-04-06 11:41:41,427 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 362 |
-
INFO 2026-04-06 11:44:46,848 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 363 |
-
INFO 2026-04-06 11:47:52,316 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 364 |
-
INFO 2026-04-06 11:50:56,954 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 365 |
-
INFO 2026-04-06 11:54:03,028 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 366 |
-
INFO 2026-04-06 11:57:07,422 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 367 |
-
INFO 2026-04-06 12:00:13,665 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 368 |
-
INFO 2026-04-06 12:03:19,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 369 |
-
INFO 2026-04-06 12:06:23,694 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 370 |
-
INFO 2026-04-06 12:09:29,445 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 371 |
-
INFO 2026-04-06 12:12:35,570 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 372 |
-
INFO 2026-04-06 12:15:41,417 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 373 |
-
INFO 2026-04-06 12:18:46,704 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 374 |
-
INFO 2026-04-06 12:21:51,861 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 375 |
-
INFO 2026-04-06 12:24:57,747 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 376 |
-
INFO 2026-04-06 12:28:01,899 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 377 |
-
INFO 2026-04-06 12:31:07,224 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 378 |
-
INFO 2026-04-06 12:34:11,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 379 |
-
INFO 2026-04-06 12:37:16,740 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 380 |
-
INFO 2026-04-06 12:40:21,216 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 381 |
-
INFO 2026-04-06 12:43:25,432 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 382 |
-
INFO 2026-04-06 12:46:31,795 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 383 |
-
INFO 2026-04-06 12:49:37,508 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 384 |
-
INFO 2026-04-06 12:52:43,064 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 385 |
-
INFO 2026-04-06 12:55:47,771 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 386 |
-
INFO 2026-04-06 12:58:53,489 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 387 |
-
INFO 2026-04-06 13:01:58,056 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 388 |
-
INFO 2026-04-06 13:05:03,713 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 389 |
-
INFO 2026-04-06 13:08:09,161 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 390 |
-
INFO 2026-04-06 13:11:13,195 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 391 |
-
INFO 2026-04-06 13:14:19,368 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 392 |
-
INFO 2026-04-06 13:17:25,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 393 |
-
INFO 2026-04-06 13:20:30,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 394 |
-
INFO 2026-04-06 13:23:35,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 395 |
-
INFO 2026-04-06 13:26:41,882 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 396 |
-
INFO 2026-04-06 13:29:48,081 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 397 |
-
INFO 2026-04-06 13:32:52,616 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 398 |
-
INFO 2026-04-06 13:35:57,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 399 |
-
INFO 2026-04-06 13:39:03,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 400 |
-
INFO 2026-04-06 13:42:10,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 401 |
-
INFO 2026-04-06 13:45:14,630 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 402 |
-
INFO 2026-04-06 13:48:20,969 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 403 |
-
INFO 2026-04-06 13:51:25,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 404 |
-
INFO 2026-04-06 13:54:30,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 405 |
-
INFO 2026-04-06 13:57:36,003 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 406 |
-
INFO 2026-04-06 14:00:41,876 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 407 |
-
INFO 2026-04-06 14:03:49,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 408 |
-
INFO 2026-04-06 14:07:04,774 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 409 |
-
INFO 2026-04-06 14:10:39,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 410 |
-
INFO 2026-04-06 14:13:44,085 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 411 |
-
INFO 2026-04-06 14:16:49,283 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 412 |
-
INFO 2026-04-06 14:19:53,528 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 413 |
-
INFO 2026-04-06 14:22:58,782 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 414 |
-
INFO 2026-04-06 14:26:03,760 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 415 |
-
INFO 2026-04-06 14:29:07,809 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 416 |
-
INFO 2026-04-06 14:32:11,851 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 417 |
-
INFO 2026-04-06 14:35:16,305 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 418 |
-
INFO 2026-04-06 14:38:22,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 419 |
-
INFO 2026-04-06 14:41:25,472 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 420 |
-
INFO 2026-04-06 14:44:30,525 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 421 |
-
INFO 2026-04-06 14:47:36,050 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 422 |
-
INFO 2026-04-06 14:50:40,765 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 423 |
-
INFO 2026-04-06 14:53:45,035 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 424 |
-
INFO 2026-04-06 14:56:50,252 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 425 |
-
INFO 2026-04-06 14:59:56,683 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 426 |
-
INFO 2026-04-06 15:03:02,324 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 427 |
-
INFO 2026-04-07 00:41:05,378 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=1, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 428 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 429 |
-
hyperparams=ConfigDict[typing.Any](
|
| 430 |
-
lr=5e-05,
|
| 431 |
-
weight_decay=0.0,
|
| 432 |
-
),
|
| 433 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 434 |
-
torch_optimizer='torch.optim.AdamW',
|
| 435 |
-
), scheduler=SequentialSchedulerConfig(
|
| 436 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 437 |
-
milestones=ConfigList[int](
|
| 438 |
-
500,
|
| 439 |
-
),
|
| 440 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 441 |
-
warmup=TorchSchedulerConfig(
|
| 442 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 443 |
-
hyperparams=ConfigDict[typing.Any](
|
| 444 |
-
start_factor=0.0002,
|
| 445 |
-
end_factor=1.0,
|
| 446 |
-
total_iters=500,
|
| 447 |
-
),
|
| 448 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 449 |
-
),
|
| 450 |
-
const=TorchSchedulerConfig(
|
| 451 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 452 |
-
hyperparams=ConfigDict[typing.Any](
|
| 453 |
-
factor=1.0,
|
| 454 |
-
total_iters=300000,
|
| 455 |
-
),
|
| 456 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 457 |
-
),
|
| 458 |
-
),
|
| 459 |
-
)
|
| 460 |
-
INFO 2026-04-07 00:41:05,381 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 461 |
-
INFO 2026-04-07 00:41:08,565 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 462 |
-
INFO 2026-04-07 00:41:17,583 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 463 |
-
INFO 2026-04-07 00:41:22,441 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 464 |
-
INFO 2026-04-07 00:41:26,525 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 465 |
-
INFO 2026-04-07 00:41:26,526 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=17960.8, dataset_name=bridge_paraphrase_k5
|
| 466 |
-
INFO 2026-04-07 00:41:26,565 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183254, rank=1
|
| 467 |
-
INFO 2026-04-07 00:41:27,562 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=1, status=start
|
| 468 |
-
INFO 2026-04-07 00:41:27,601 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183254, num_data_pointers=156074, pct=85.17%, rank=1, status=success
|
| 469 |
-
INFO 2026-04-07 00:41:27,602 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=22221.5
|
| 470 |
-
INFO 2026-04-07 00:41:27,607 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=1, status=start
|
| 471 |
-
INFO 2026-04-07 00:43:48,421 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=1, status=success
|
| 472 |
-
INFO 2026-04-07 00:44:10,421 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=1
|
| 473 |
-
INFO 2026-04-07 00:44:10,423 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=1, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 474 |
-
INFO 2026-04-07 00:44:10,488 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=3313510, parent_pid=3310179, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 475 |
-
INFO 2026-04-07 00:44:10,498 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=1, hostname=msp3-5, worker_pid=3313518, parent_pid=3310179, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 476 |
-
INFO 2026-04-07 00:47:36,029 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 477 |
-
INFO 2026-04-07 00:50:38,869 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 478 |
-
INFO 2026-04-07 00:53:42,313 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 479 |
-
INFO 2026-04-07 00:56:45,086 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 480 |
-
INFO 2026-04-07 00:59:47,986 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 481 |
-
INFO 2026-04-07 01:02:50,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 482 |
-
INFO 2026-04-07 01:05:53,527 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 483 |
-
INFO 2026-04-07 01:08:56,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 484 |
-
INFO 2026-04-07 01:11:58,574 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 485 |
-
INFO 2026-04-07 01:15:02,202 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 486 |
-
INFO 2026-04-07 01:18:05,103 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 487 |
-
INFO 2026-04-07 01:21:08,473 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 488 |
-
INFO 2026-04-07 01:24:11,699 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 489 |
-
INFO 2026-04-07 01:27:15,143 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 490 |
-
INFO 2026-04-07 01:30:16,884 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 491 |
-
INFO 2026-04-07 01:33:19,435 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 492 |
-
INFO 2026-04-07 01:36:22,455 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 493 |
-
INFO 2026-04-07 01:39:25,142 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 494 |
-
INFO 2026-04-07 01:42:29,620 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 495 |
-
INFO 2026-04-07 01:45:32,634 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 496 |
-
INFO 2026-04-07 01:48:36,837 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 497 |
-
INFO 2026-04-07 01:51:39,490 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 498 |
-
INFO 2026-04-07 01:54:42,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 499 |
-
INFO 2026-04-07 01:57:46,410 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 500 |
-
INFO 2026-04-07 02:00:50,170 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 501 |
-
INFO 2026-04-07 02:03:54,279 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 502 |
-
INFO 2026-04-07 02:06:57,975 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 503 |
-
INFO 2026-04-07 02:10:02,504 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 504 |
-
INFO 2026-04-07 02:13:05,975 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 505 |
-
INFO 2026-04-07 02:16:10,140 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 506 |
-
INFO 2026-04-07 02:19:14,240 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 507 |
-
INFO 2026-04-07 02:22:17,137 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 508 |
-
INFO 2026-04-07 02:25:21,416 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 509 |
-
INFO 2026-04-07 02:28:24,738 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 510 |
-
INFO 2026-04-07 02:31:29,706 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 511 |
-
INFO 2026-04-07 02:34:33,435 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 512 |
-
INFO 2026-04-07 02:37:37,740 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 513 |
-
INFO 2026-04-07 02:40:41,387 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 514 |
-
INFO 2026-04-07 02:43:44,863 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 515 |
-
INFO 2026-04-07 02:46:48,385 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 516 |
-
INFO 2026-04-07 02:49:52,438 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 517 |
-
INFO 2026-04-07 02:52:56,831 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 518 |
-
INFO 2026-04-07 02:56:01,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 519 |
-
INFO 2026-04-07 02:59:07,330 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 520 |
-
INFO 2026-04-07 03:02:11,725 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 521 |
-
INFO 2026-04-07 03:05:16,725 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 522 |
-
INFO 2026-04-07 03:08:20,098 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 523 |
-
INFO 2026-04-07 03:11:23,979 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 524 |
-
INFO 2026-04-07 03:14:28,686 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 525 |
-
INFO 2026-04-07 03:17:51,601 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 526 |
-
INFO 2026-04-07 03:18:34,221 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=1
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7792d124f680eb3b1187797c9c324f23e4cf0f643d2787788905cd7f2b83bf2d
|
| 3 |
+
size 63999
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_2.log
CHANGED
|
@@ -1,526 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
lr=5e-05,
|
| 5 |
-
weight_decay=0.0,
|
| 6 |
-
),
|
| 7 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 8 |
-
torch_optimizer='torch.optim.AdamW',
|
| 9 |
-
), scheduler=SequentialSchedulerConfig(
|
| 10 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 11 |
-
milestones=ConfigList[int](
|
| 12 |
-
500,
|
| 13 |
-
),
|
| 14 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 15 |
-
warmup=TorchSchedulerConfig(
|
| 16 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 17 |
-
hyperparams=ConfigDict[typing.Any](
|
| 18 |
-
start_factor=0.0002,
|
| 19 |
-
end_factor=1.0,
|
| 20 |
-
total_iters=500,
|
| 21 |
-
),
|
| 22 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 23 |
-
),
|
| 24 |
-
const=TorchSchedulerConfig(
|
| 25 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 26 |
-
hyperparams=ConfigDict[typing.Any](
|
| 27 |
-
factor=1.0,
|
| 28 |
-
total_iters=300000,
|
| 29 |
-
),
|
| 30 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 31 |
-
),
|
| 32 |
-
),
|
| 33 |
-
)
|
| 34 |
-
INFO 2026-04-05 07:47:39,890 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 35 |
-
INFO 2026-04-05 07:47:42,059 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge
|
| 36 |
-
INFO 2026-04-05 07:47:55,884 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 37 |
-
INFO 2026-04-05 07:48:00,080 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 38 |
-
INFO 2026-04-05 07:48:00,080 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=18021.2, dataset_name=bridge
|
| 39 |
-
INFO 2026-04-05 07:48:00,119 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183271, rank=2
|
| 40 |
-
INFO 2026-04-05 07:48:01,165 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge rank=2, status=start
|
| 41 |
-
INFO 2026-04-05 07:48:01,203 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
|
| 42 |
-
INFO 2026-04-05 07:48:01,205 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=21314.9
|
| 43 |
-
INFO 2026-04-05 07:48:34,458 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=2
|
| 44 |
-
INFO 2026-04-05 07:48:34,460 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 45 |
-
INFO 2026-04-05 07:48:34,538 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-2, worker_pid=3875756, parent_pid=3872891, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 46 |
-
INFO 2026-04-05 07:48:34,582 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-2, worker_pid=3875804, parent_pid=3872891, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 47 |
-
INFO 2026-04-05 07:51:52,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 48 |
-
INFO 2026-04-05 07:54:44,997 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 49 |
-
INFO 2026-04-05 07:57:35,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 50 |
-
INFO 2026-04-05 08:00:26,862 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 51 |
-
INFO 2026-04-05 08:03:18,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 52 |
-
INFO 2026-04-05 08:06:09,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 53 |
-
INFO 2026-04-05 08:09:01,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 54 |
-
INFO 2026-04-05 08:11:53,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 55 |
-
INFO 2026-04-05 08:14:44,578 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 56 |
-
INFO 2026-04-05 08:17:37,375 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 57 |
-
INFO 2026-04-05 08:20:30,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 58 |
-
INFO 2026-04-05 08:23:23,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 59 |
-
INFO 2026-04-05 08:26:15,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 60 |
-
INFO 2026-04-05 08:29:07,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 61 |
-
INFO 2026-04-05 08:31:59,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 62 |
-
INFO 2026-04-05 08:34:50,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 63 |
-
INFO 2026-04-05 08:37:43,456 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 64 |
-
INFO 2026-04-05 08:40:35,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 65 |
-
INFO 2026-04-05 08:43:28,467 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 66 |
-
INFO 2026-04-05 08:46:21,132 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 67 |
-
INFO 2026-04-05 08:49:13,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 68 |
-
INFO 2026-04-05 08:52:04,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 69 |
-
INFO 2026-04-05 08:54:56,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 70 |
-
INFO 2026-04-05 08:57:48,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 71 |
-
INFO 2026-04-05 09:00:40,238 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 72 |
-
INFO 2026-04-05 09:03:32,325 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 73 |
-
INFO 2026-04-05 09:06:23,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 74 |
-
INFO 2026-04-05 09:09:15,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 75 |
-
INFO 2026-04-05 09:12:08,117 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 76 |
-
INFO 2026-04-05 09:14:59,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 77 |
-
INFO 2026-04-05 09:17:51,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 78 |
-
INFO 2026-04-05 09:20:42,298 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 79 |
-
INFO 2026-04-05 09:23:34,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 80 |
-
INFO 2026-04-05 09:26:26,063 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 81 |
-
INFO 2026-04-05 09:29:18,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 82 |
-
INFO 2026-04-05 09:32:09,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 83 |
-
INFO 2026-04-05 09:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 84 |
-
INFO 2026-04-05 09:37:52,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 85 |
-
INFO 2026-04-05 09:40:43,428 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 86 |
-
INFO 2026-04-05 09:43:34,586 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 87 |
-
INFO 2026-04-05 09:46:26,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 88 |
-
INFO 2026-04-05 09:49:17,267 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 89 |
-
INFO 2026-04-05 09:52:06,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 90 |
-
INFO 2026-04-05 09:54:57,889 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 91 |
-
INFO 2026-04-05 09:57:48,894 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 92 |
-
INFO 2026-04-05 10:00:41,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 93 |
-
INFO 2026-04-05 10:03:33,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 94 |
-
INFO 2026-04-05 10:06:24,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 95 |
-
INFO 2026-04-05 10:09:17,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 96 |
-
INFO 2026-04-05 10:12:18,686 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 97 |
-
INFO 2026-04-05 10:15:33,482 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 98 |
-
INFO 2026-04-05 10:18:24,744 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 99 |
-
INFO 2026-04-05 10:21:17,302 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 100 |
-
INFO 2026-04-05 10:24:08,917 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 101 |
-
INFO 2026-04-05 10:27:00,005 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 102 |
-
INFO 2026-04-05 10:29:51,958 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 103 |
-
INFO 2026-04-05 10:32:42,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 104 |
-
INFO 2026-04-05 10:35:34,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 105 |
-
INFO 2026-04-05 10:38:25,583 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 106 |
-
INFO 2026-04-05 10:41:18,350 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 107 |
-
INFO 2026-04-05 10:44:08,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 108 |
-
INFO 2026-04-05 10:47:00,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 109 |
-
INFO 2026-04-05 10:49:51,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 110 |
-
INFO 2026-04-05 10:52:43,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 111 |
-
INFO 2026-04-05 10:55:35,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 112 |
-
INFO 2026-04-05 10:58:26,949 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 113 |
-
INFO 2026-04-05 11:01:18,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 114 |
-
INFO 2026-04-05 11:04:10,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 115 |
-
INFO 2026-04-05 11:07:02,461 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 116 |
-
INFO 2026-04-05 11:09:53,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 117 |
-
INFO 2026-04-05 11:12:45,451 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 118 |
-
INFO 2026-04-05 11:15:36,492 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 119 |
-
INFO 2026-04-05 11:18:28,400 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 120 |
-
INFO 2026-04-05 11:21:20,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 121 |
-
INFO 2026-04-05 11:24:11,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 122 |
-
INFO 2026-04-05 11:27:02,198 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 123 |
-
INFO 2026-04-05 11:29:53,736 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 124 |
-
INFO 2026-04-05 11:32:44,011 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 125 |
-
INFO 2026-04-05 11:35:35,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 126 |
-
INFO 2026-04-05 11:38:25,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 127 |
-
INFO 2026-04-05 11:41:15,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 128 |
-
INFO 2026-04-05 11:44:05,134 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 129 |
-
INFO 2026-04-05 11:46:56,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 130 |
-
INFO 2026-04-05 11:49:47,469 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 131 |
-
INFO 2026-04-05 11:52:38,563 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 132 |
-
INFO 2026-04-05 11:55:30,290 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 133 |
-
INFO 2026-04-05 11:58:21,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 134 |
-
INFO 2026-04-05 12:01:13,424 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 135 |
-
INFO 2026-04-05 12:04:04,015 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 136 |
-
INFO 2026-04-05 12:06:55,195 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 137 |
-
INFO 2026-04-05 12:09:46,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 138 |
-
INFO 2026-04-05 12:12:37,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 139 |
-
INFO 2026-04-05 12:15:28,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 140 |
-
INFO 2026-04-05 12:18:19,903 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 141 |
-
INFO 2026-04-05 12:21:11,565 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 142 |
-
INFO 2026-04-05 12:24:02,399 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 143 |
-
INFO 2026-04-05 12:26:54,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 144 |
-
INFO 2026-04-05 12:29:45,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 145 |
-
INFO 2026-04-05 12:32:36,286 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 146 |
-
INFO 2026-04-05 12:35:36,776 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 147 |
-
INFO 2026-04-05 12:38:50,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 148 |
-
INFO 2026-04-05 12:41:41,343 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 149 |
-
INFO 2026-04-05 12:44:32,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 150 |
-
INFO 2026-04-05 12:47:23,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 151 |
-
INFO 2026-04-05 12:50:13,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 152 |
-
INFO 2026-04-05 12:53:06,274 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 153 |
-
INFO 2026-04-05 12:55:56,979 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 154 |
-
INFO 2026-04-05 12:58:48,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 155 |
-
INFO 2026-04-05 13:01:39,887 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 156 |
-
INFO 2026-04-05 13:04:31,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 157 |
-
INFO 2026-04-05 13:07:22,319 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 158 |
-
INFO 2026-04-05 13:10:15,398 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 159 |
-
INFO 2026-04-05 13:13:06,519 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 160 |
-
INFO 2026-04-05 13:15:57,164 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 161 |
-
INFO 2026-04-05 13:18:49,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 162 |
-
INFO 2026-04-05 13:21:40,162 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 163 |
-
INFO 2026-04-05 13:24:31,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 164 |
-
INFO 2026-04-05 13:27:22,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 165 |
-
INFO 2026-04-05 13:30:13,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 166 |
-
INFO 2026-04-05 13:33:06,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 167 |
-
INFO 2026-04-05 13:35:57,606 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 168 |
-
INFO 2026-04-05 13:38:47,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 169 |
-
INFO 2026-04-05 13:41:40,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 170 |
-
INFO 2026-04-05 13:44:31,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 171 |
-
INFO 2026-04-05 13:47:23,254 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 172 |
-
INFO 2026-04-05 13:50:14,660 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 173 |
-
INFO 2026-04-05 13:53:06,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 174 |
-
INFO 2026-04-05 13:55:58,107 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 175 |
-
INFO 2026-04-05 13:58:50,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 176 |
-
INFO 2026-04-05 14:01:41,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 177 |
-
INFO 2026-04-05 14:04:32,278 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 178 |
-
INFO 2026-04-05 14:07:24,300 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 179 |
-
INFO 2026-04-05 14:10:15,785 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 180 |
-
INFO 2026-04-05 14:13:07,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 181 |
-
INFO 2026-04-05 14:15:58,762 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 182 |
-
INFO 2026-04-05 14:18:49,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 183 |
-
INFO 2026-04-05 14:21:40,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 184 |
-
INFO 2026-04-05 14:24:31,803 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 185 |
-
INFO 2026-04-05 14:27:22,607 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 186 |
-
INFO 2026-04-05 14:30:14,379 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 187 |
-
INFO 2026-04-05 14:33:06,049 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 188 |
-
INFO 2026-04-05 14:35:56,094 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 189 |
-
INFO 2026-04-05 14:38:48,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 190 |
-
INFO 2026-04-05 14:41:40,352 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 191 |
-
INFO 2026-04-05 14:44:32,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 192 |
-
INFO 2026-04-05 14:47:25,214 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 193 |
-
INFO 2026-04-05 14:50:15,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 194 |
-
INFO 2026-04-05 14:53:07,347 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 195 |
-
INFO 2026-04-05 14:56:00,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 196 |
-
INFO 2026-04-05 14:59:02,183 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 197 |
-
INFO 2026-04-05 14:59:35,376 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=2
|
| 198 |
-
INFO 2026-04-06 07:33:53,119 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=2, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 199 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 200 |
-
hyperparams=ConfigDict[typing.Any](
|
| 201 |
-
lr=5e-05,
|
| 202 |
-
weight_decay=0.0,
|
| 203 |
-
),
|
| 204 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 205 |
-
torch_optimizer='torch.optim.AdamW',
|
| 206 |
-
), scheduler=SequentialSchedulerConfig(
|
| 207 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 208 |
-
milestones=ConfigList[int](
|
| 209 |
-
500,
|
| 210 |
-
),
|
| 211 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 212 |
-
warmup=TorchSchedulerConfig(
|
| 213 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 214 |
-
hyperparams=ConfigDict[typing.Any](
|
| 215 |
-
start_factor=0.0002,
|
| 216 |
-
end_factor=1.0,
|
| 217 |
-
total_iters=500,
|
| 218 |
-
),
|
| 219 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 220 |
-
),
|
| 221 |
-
const=TorchSchedulerConfig(
|
| 222 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 223 |
-
hyperparams=ConfigDict[typing.Any](
|
| 224 |
-
factor=1.0,
|
| 225 |
-
total_iters=300000,
|
| 226 |
-
),
|
| 227 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 228 |
-
),
|
| 229 |
-
),
|
| 230 |
-
)
|
| 231 |
-
INFO 2026-04-06 07:33:53,122 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 232 |
-
INFO 2026-04-06 07:33:56,084 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 233 |
-
INFO 2026-04-06 07:34:13,383 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 234 |
-
INFO 2026-04-06 07:34:20,583 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 235 |
-
INFO 2026-04-06 07:34:25,333 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 236 |
-
INFO 2026-04-06 07:34:25,333 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=29248.8, dataset_name=bridge_paraphrase_k5
|
| 237 |
-
INFO 2026-04-06 07:34:25,374 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183271, rank=2
|
| 238 |
-
INFO 2026-04-06 07:34:26,288 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=2, status=start
|
| 239 |
-
INFO 2026-04-06 07:34:26,333 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
|
| 240 |
-
INFO 2026-04-06 07:34:26,334 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=33212.6
|
| 241 |
-
INFO 2026-04-06 07:34:26,340 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=2, status=start
|
| 242 |
-
INFO 2026-04-06 07:36:05,842 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=2, status=success
|
| 243 |
-
INFO 2026-04-06 07:36:27,876 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=2
|
| 244 |
-
INFO 2026-04-06 07:36:27,879 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 245 |
-
INFO 2026-04-06 07:36:27,947 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=2465174, parent_pid=2461887, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 246 |
-
INFO 2026-04-06 07:36:28,002 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=2465214, parent_pid=2461887, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 247 |
-
INFO 2026-04-06 07:39:43,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 248 |
-
INFO 2026-04-06 07:42:35,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 249 |
-
INFO 2026-04-06 07:45:26,710 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 250 |
-
INFO 2026-04-06 07:48:18,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 251 |
-
INFO 2026-04-06 07:51:08,588 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 252 |
-
INFO 2026-04-06 07:54:00,463 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 253 |
-
INFO 2026-04-06 07:56:51,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 254 |
-
INFO 2026-04-06 07:59:43,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 255 |
-
INFO 2026-04-06 08:02:34,655 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 256 |
-
INFO 2026-04-06 08:05:26,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 257 |
-
INFO 2026-04-06 08:08:18,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 258 |
-
INFO 2026-04-06 08:11:11,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 259 |
-
INFO 2026-04-06 08:14:03,149 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 260 |
-
INFO 2026-04-06 08:16:57,119 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 261 |
-
INFO 2026-04-06 08:19:48,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 262 |
-
INFO 2026-04-06 08:22:39,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 263 |
-
INFO 2026-04-06 08:25:31,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 264 |
-
INFO 2026-04-06 08:28:25,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 265 |
-
INFO 2026-04-06 08:31:16,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 266 |
-
INFO 2026-04-06 08:34:09,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 267 |
-
INFO 2026-04-06 08:37:01,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 268 |
-
INFO 2026-04-06 08:39:53,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 269 |
-
INFO 2026-04-06 08:42:45,381 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 270 |
-
INFO 2026-04-06 08:45:38,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 271 |
-
INFO 2026-04-06 08:48:30,988 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 272 |
-
INFO 2026-04-06 08:51:23,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 273 |
-
INFO 2026-04-06 08:54:13,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 274 |
-
INFO 2026-04-06 08:57:05,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 275 |
-
INFO 2026-04-06 08:59:57,948 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 276 |
-
INFO 2026-04-06 09:02:49,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 277 |
-
INFO 2026-04-06 09:05:41,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 278 |
-
INFO 2026-04-06 09:08:33,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 279 |
-
INFO 2026-04-06 09:11:25,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 280 |
-
INFO 2026-04-06 09:14:17,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 281 |
-
INFO 2026-04-06 09:17:08,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 282 |
-
INFO 2026-04-06 09:20:01,792 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 283 |
-
INFO 2026-04-06 09:22:53,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 284 |
-
INFO 2026-04-06 09:25:46,358 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 285 |
-
INFO 2026-04-06 09:28:40,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 286 |
-
INFO 2026-04-06 09:31:33,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 287 |
-
INFO 2026-04-06 09:34:24,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 288 |
-
INFO 2026-04-06 09:37:17,501 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 289 |
-
INFO 2026-04-06 09:40:08,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 290 |
-
INFO 2026-04-06 09:43:00,698 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 291 |
-
INFO 2026-04-06 09:45:52,659 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 292 |
-
INFO 2026-04-06 09:48:44,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 293 |
-
INFO 2026-04-06 09:51:36,495 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 294 |
-
INFO 2026-04-06 09:54:29,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 295 |
-
INFO 2026-04-06 09:57:20,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 296 |
-
INFO 2026-04-06 10:00:22,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 297 |
-
INFO 2026-04-06 10:03:48,881 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 298 |
-
INFO 2026-04-06 10:06:52,081 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 299 |
-
INFO 2026-04-06 10:09:59,023 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 300 |
-
INFO 2026-04-06 10:13:05,837 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 301 |
-
INFO 2026-04-06 10:16:11,929 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 302 |
-
INFO 2026-04-06 10:19:15,977 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 303 |
-
INFO 2026-04-06 10:22:24,664 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 304 |
-
INFO 2026-04-06 10:25:33,549 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 305 |
-
INFO 2026-04-06 10:28:42,172 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 306 |
-
INFO 2026-04-06 10:31:51,516 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 307 |
-
INFO 2026-04-06 10:35:00,444 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 308 |
-
INFO 2026-04-06 10:38:08,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 309 |
-
INFO 2026-04-06 10:41:15,601 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 310 |
-
INFO 2026-04-06 11:29:31,533 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=2, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 311 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 312 |
-
hyperparams=ConfigDict[typing.Any](
|
| 313 |
-
lr=5e-05,
|
| 314 |
-
weight_decay=0.0,
|
| 315 |
-
),
|
| 316 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 317 |
-
torch_optimizer='torch.optim.AdamW',
|
| 318 |
-
), scheduler=SequentialSchedulerConfig(
|
| 319 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 320 |
-
milestones=ConfigList[int](
|
| 321 |
-
500,
|
| 322 |
-
),
|
| 323 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 324 |
-
warmup=TorchSchedulerConfig(
|
| 325 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 326 |
-
hyperparams=ConfigDict[typing.Any](
|
| 327 |
-
start_factor=0.0002,
|
| 328 |
-
end_factor=1.0,
|
| 329 |
-
total_iters=500,
|
| 330 |
-
),
|
| 331 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 332 |
-
),
|
| 333 |
-
const=TorchSchedulerConfig(
|
| 334 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 335 |
-
hyperparams=ConfigDict[typing.Any](
|
| 336 |
-
factor=1.0,
|
| 337 |
-
total_iters=300000,
|
| 338 |
-
),
|
| 339 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 340 |
-
),
|
| 341 |
-
),
|
| 342 |
-
)
|
| 343 |
-
INFO 2026-04-06 11:29:31,537 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 344 |
-
INFO 2026-04-06 11:29:34,940 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 345 |
-
INFO 2026-04-06 11:29:43,630 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 346 |
-
INFO 2026-04-06 11:29:47,887 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 347 |
-
INFO 2026-04-06 11:29:51,559 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 348 |
-
INFO 2026-04-06 11:29:51,559 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=16619.3, dataset_name=bridge_paraphrase_k5
|
| 349 |
-
INFO 2026-04-06 11:29:51,600 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183271, rank=2
|
| 350 |
-
INFO 2026-04-06 11:29:52,408 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=2, status=start
|
| 351 |
-
INFO 2026-04-06 11:29:52,447 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
|
| 352 |
-
INFO 2026-04-06 11:29:52,448 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=20911.5
|
| 353 |
-
INFO 2026-04-06 11:29:52,453 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=2, status=start
|
| 354 |
-
INFO 2026-04-06 11:31:39,838 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=2, status=success
|
| 355 |
-
INFO 2026-04-06 11:32:01,842 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=2
|
| 356 |
-
INFO 2026-04-06 11:32:01,844 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 357 |
-
INFO 2026-04-06 11:32:02,006 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-7, worker_pid=3492777, parent_pid=3489442, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 358 |
-
INFO 2026-04-06 11:32:02,026 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-7, worker_pid=3492786, parent_pid=3489442, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 359 |
-
INFO 2026-04-06 11:35:29,328 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 360 |
-
INFO 2026-04-06 11:38:35,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 361 |
-
INFO 2026-04-06 11:41:41,427 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 362 |
-
INFO 2026-04-06 11:44:46,852 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 363 |
-
INFO 2026-04-06 11:47:52,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 364 |
-
INFO 2026-04-06 11:50:56,952 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 365 |
-
INFO 2026-04-06 11:54:03,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 366 |
-
INFO 2026-04-06 11:57:07,420 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 367 |
-
INFO 2026-04-06 12:00:13,670 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 368 |
-
INFO 2026-04-06 12:03:19,351 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 369 |
-
INFO 2026-04-06 12:06:23,695 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 370 |
-
INFO 2026-04-06 12:09:29,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 371 |
-
INFO 2026-04-06 12:12:35,569 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 372 |
-
INFO 2026-04-06 12:15:41,416 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 373 |
-
INFO 2026-04-06 12:18:46,702 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 374 |
-
INFO 2026-04-06 12:21:51,861 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 375 |
-
INFO 2026-04-06 12:24:57,746 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 376 |
-
INFO 2026-04-06 12:28:01,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 377 |
-
INFO 2026-04-06 12:31:07,224 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 378 |
-
INFO 2026-04-06 12:34:11,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 379 |
-
INFO 2026-04-06 12:37:16,738 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 380 |
-
INFO 2026-04-06 12:40:21,215 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 381 |
-
INFO 2026-04-06 12:43:25,432 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 382 |
-
INFO 2026-04-06 12:46:31,790 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 383 |
-
INFO 2026-04-06 12:49:37,505 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 384 |
-
INFO 2026-04-06 12:52:43,061 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 385 |
-
INFO 2026-04-06 12:55:47,771 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 386 |
-
INFO 2026-04-06 12:58:53,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 387 |
-
INFO 2026-04-06 13:01:58,051 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 388 |
-
INFO 2026-04-06 13:05:03,711 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 389 |
-
INFO 2026-04-06 13:08:09,155 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 390 |
-
INFO 2026-04-06 13:11:13,195 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 391 |
-
INFO 2026-04-06 13:14:19,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 392 |
-
INFO 2026-04-06 13:17:25,217 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 393 |
-
INFO 2026-04-06 13:20:30,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 394 |
-
INFO 2026-04-06 13:23:35,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 395 |
-
INFO 2026-04-06 13:26:41,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 396 |
-
INFO 2026-04-06 13:29:48,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 397 |
-
INFO 2026-04-06 13:32:52,616 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 398 |
-
INFO 2026-04-06 13:35:57,628 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 399 |
-
INFO 2026-04-06 13:39:03,078 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 400 |
-
INFO 2026-04-06 13:42:10,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 401 |
-
INFO 2026-04-06 13:45:14,630 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 402 |
-
INFO 2026-04-06 13:48:20,967 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 403 |
-
INFO 2026-04-06 13:51:25,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 404 |
-
INFO 2026-04-06 13:54:30,720 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 405 |
-
INFO 2026-04-06 13:57:36,001 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 406 |
-
INFO 2026-04-06 14:00:41,873 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 407 |
-
INFO 2026-04-06 14:03:49,030 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 408 |
-
INFO 2026-04-06 14:07:04,710 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 409 |
-
INFO 2026-04-06 14:10:39,270 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 410 |
-
INFO 2026-04-06 14:13:44,082 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 411 |
-
INFO 2026-04-06 14:16:49,280 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 412 |
-
INFO 2026-04-06 14:19:53,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 413 |
-
INFO 2026-04-06 14:22:58,775 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 414 |
-
INFO 2026-04-06 14:26:03,758 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 415 |
-
INFO 2026-04-06 14:29:07,810 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 416 |
-
INFO 2026-04-06 14:32:11,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 417 |
-
INFO 2026-04-06 14:35:16,304 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 418 |
-
INFO 2026-04-06 14:38:22,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 419 |
-
INFO 2026-04-06 14:41:25,471 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 420 |
-
INFO 2026-04-06 14:44:30,524 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 421 |
-
INFO 2026-04-06 14:47:36,041 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 422 |
-
INFO 2026-04-06 14:50:40,764 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 423 |
-
INFO 2026-04-06 14:53:45,031 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 424 |
-
INFO 2026-04-06 14:56:50,250 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 425 |
-
INFO 2026-04-06 14:59:56,683 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 426 |
-
INFO 2026-04-06 15:03:02,331 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 427 |
-
INFO 2026-04-07 00:41:05,445 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=2, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 428 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 429 |
-
hyperparams=ConfigDict[typing.Any](
|
| 430 |
-
lr=5e-05,
|
| 431 |
-
weight_decay=0.0,
|
| 432 |
-
),
|
| 433 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 434 |
-
torch_optimizer='torch.optim.AdamW',
|
| 435 |
-
), scheduler=SequentialSchedulerConfig(
|
| 436 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 437 |
-
milestones=ConfigList[int](
|
| 438 |
-
500,
|
| 439 |
-
),
|
| 440 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 441 |
-
warmup=TorchSchedulerConfig(
|
| 442 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 443 |
-
hyperparams=ConfigDict[typing.Any](
|
| 444 |
-
start_factor=0.0002,
|
| 445 |
-
end_factor=1.0,
|
| 446 |
-
total_iters=500,
|
| 447 |
-
),
|
| 448 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 449 |
-
),
|
| 450 |
-
const=TorchSchedulerConfig(
|
| 451 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 452 |
-
hyperparams=ConfigDict[typing.Any](
|
| 453 |
-
factor=1.0,
|
| 454 |
-
total_iters=300000,
|
| 455 |
-
),
|
| 456 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 457 |
-
),
|
| 458 |
-
),
|
| 459 |
-
)
|
| 460 |
-
INFO 2026-04-07 00:41:05,450 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 461 |
-
INFO 2026-04-07 00:41:08,566 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 462 |
-
INFO 2026-04-07 00:41:17,724 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 463 |
-
INFO 2026-04-07 00:41:22,571 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 464 |
-
INFO 2026-04-07 00:41:26,705 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 465 |
-
INFO 2026-04-07 00:41:26,705 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=18138.6, dataset_name=bridge_paraphrase_k5
|
| 466 |
-
INFO 2026-04-07 00:41:26,745 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183271, rank=2
|
| 467 |
-
INFO 2026-04-07 00:41:27,734 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=2, status=start
|
| 468 |
-
INFO 2026-04-07 00:41:27,773 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183271, num_data_pointers=166095, pct=90.63%, rank=2, status=success
|
| 469 |
-
INFO 2026-04-07 00:41:27,774 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=22324.7
|
| 470 |
-
INFO 2026-04-07 00:41:27,779 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=2, status=start
|
| 471 |
-
INFO 2026-04-07 00:43:48,420 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=2, status=success
|
| 472 |
-
INFO 2026-04-07 00:44:10,421 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=2
|
| 473 |
-
INFO 2026-04-07 00:44:10,423 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=2, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 474 |
-
INFO 2026-04-07 00:44:10,474 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=3313509, parent_pid=3310180, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 475 |
-
INFO 2026-04-07 00:44:10,497 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=2, hostname=msp3-5, worker_pid=3313514, parent_pid=3310180, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 476 |
-
INFO 2026-04-07 00:47:36,025 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 477 |
-
INFO 2026-04-07 00:50:38,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 478 |
-
INFO 2026-04-07 00:53:42,312 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 479 |
-
INFO 2026-04-07 00:56:45,082 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 480 |
-
INFO 2026-04-07 00:59:47,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 481 |
-
INFO 2026-04-07 01:02:50,207 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 482 |
-
INFO 2026-04-07 01:05:53,527 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 483 |
-
INFO 2026-04-07 01:08:56,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 484 |
-
INFO 2026-04-07 01:11:58,574 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 485 |
-
INFO 2026-04-07 01:15:02,202 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 486 |
-
INFO 2026-04-07 01:18:05,103 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 487 |
-
INFO 2026-04-07 01:21:08,472 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 488 |
-
INFO 2026-04-07 01:24:11,705 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 489 |
-
INFO 2026-04-07 01:27:15,142 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 490 |
-
INFO 2026-04-07 01:30:16,884 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 491 |
-
INFO 2026-04-07 01:33:19,433 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 492 |
-
INFO 2026-04-07 01:36:22,454 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 493 |
-
INFO 2026-04-07 01:39:25,140 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 494 |
-
INFO 2026-04-07 01:42:29,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 495 |
-
INFO 2026-04-07 01:45:32,636 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 496 |
-
INFO 2026-04-07 01:48:36,837 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 497 |
-
INFO 2026-04-07 01:51:39,489 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 498 |
-
INFO 2026-04-07 01:54:42,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 499 |
-
INFO 2026-04-07 01:57:46,408 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 500 |
-
INFO 2026-04-07 02:00:50,168 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 501 |
-
INFO 2026-04-07 02:03:54,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 502 |
-
INFO 2026-04-07 02:06:57,974 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 503 |
-
INFO 2026-04-07 02:10:02,506 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 504 |
-
INFO 2026-04-07 02:13:05,974 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 505 |
-
INFO 2026-04-07 02:16:10,139 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 506 |
-
INFO 2026-04-07 02:19:14,241 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 507 |
-
INFO 2026-04-07 02:22:17,137 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 508 |
-
INFO 2026-04-07 02:25:21,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 509 |
-
INFO 2026-04-07 02:28:24,735 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 510 |
-
INFO 2026-04-07 02:31:29,705 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 511 |
-
INFO 2026-04-07 02:34:33,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 512 |
-
INFO 2026-04-07 02:37:37,738 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 513 |
-
INFO 2026-04-07 02:40:41,387 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 514 |
-
INFO 2026-04-07 02:43:44,849 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 515 |
-
INFO 2026-04-07 02:46:48,384 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 516 |
-
INFO 2026-04-07 02:49:52,436 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 517 |
-
INFO 2026-04-07 02:52:56,831 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 518 |
-
INFO 2026-04-07 02:56:01,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 519 |
-
INFO 2026-04-07 02:59:07,327 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 520 |
-
INFO 2026-04-07 03:02:11,724 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 521 |
-
INFO 2026-04-07 03:05:16,726 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 522 |
-
INFO 2026-04-07 03:08:20,100 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 523 |
-
INFO 2026-04-07 03:11:23,978 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 524 |
-
INFO 2026-04-07 03:14:28,686 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 525 |
-
INFO 2026-04-07 03:17:51,379 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 526 |
-
INFO 2026-04-07 03:18:34,222 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=2
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aaa6984aa5db79c0d8b484f905b51cc55b5213c8c95420d7d614c9d54f45fd90
|
| 3 |
+
size 63999
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_3.log
CHANGED
|
@@ -1,526 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
lr=5e-05,
|
| 5 |
-
weight_decay=0.0,
|
| 6 |
-
),
|
| 7 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 8 |
-
torch_optimizer='torch.optim.AdamW',
|
| 9 |
-
), scheduler=SequentialSchedulerConfig(
|
| 10 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 11 |
-
milestones=ConfigList[int](
|
| 12 |
-
500,
|
| 13 |
-
),
|
| 14 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 15 |
-
warmup=TorchSchedulerConfig(
|
| 16 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 17 |
-
hyperparams=ConfigDict[typing.Any](
|
| 18 |
-
start_factor=0.0002,
|
| 19 |
-
end_factor=1.0,
|
| 20 |
-
total_iters=500,
|
| 21 |
-
),
|
| 22 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 23 |
-
),
|
| 24 |
-
const=TorchSchedulerConfig(
|
| 25 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 26 |
-
hyperparams=ConfigDict[typing.Any](
|
| 27 |
-
factor=1.0,
|
| 28 |
-
total_iters=300000,
|
| 29 |
-
),
|
| 30 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 31 |
-
),
|
| 32 |
-
),
|
| 33 |
-
)
|
| 34 |
-
INFO 2026-04-05 07:47:39,921 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 35 |
-
INFO 2026-04-05 07:47:42,035 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge
|
| 36 |
-
INFO 2026-04-05 07:48:03,688 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 37 |
-
INFO 2026-04-05 07:48:08,057 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 38 |
-
INFO 2026-04-05 07:48:08,057 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=26022.5, dataset_name=bridge
|
| 39 |
-
INFO 2026-04-05 07:48:08,097 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183231, rank=3
|
| 40 |
-
INFO 2026-04-05 07:48:08,968 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge rank=3, status=start
|
| 41 |
-
INFO 2026-04-05 07:48:09,007 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
|
| 42 |
-
INFO 2026-04-05 07:48:09,009 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=29087.800000000003
|
| 43 |
-
INFO 2026-04-05 07:48:34,458 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=3
|
| 44 |
-
INFO 2026-04-05 07:48:34,460 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 45 |
-
INFO 2026-04-05 07:48:34,516 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-2, worker_pid=3875749, parent_pid=3872892, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 46 |
-
INFO 2026-04-05 07:48:34,547 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-2, worker_pid=3875761, parent_pid=3872892, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 47 |
-
INFO 2026-04-05 07:51:52,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 48 |
-
INFO 2026-04-05 07:54:44,997 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 49 |
-
INFO 2026-04-05 07:57:35,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 50 |
-
INFO 2026-04-05 08:00:26,862 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 51 |
-
INFO 2026-04-05 08:03:18,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 52 |
-
INFO 2026-04-05 08:06:09,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 53 |
-
INFO 2026-04-05 08:09:01,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 54 |
-
INFO 2026-04-05 08:11:53,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 55 |
-
INFO 2026-04-05 08:14:44,578 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 56 |
-
INFO 2026-04-05 08:17:37,375 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 57 |
-
INFO 2026-04-05 08:20:30,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 58 |
-
INFO 2026-04-05 08:23:23,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 59 |
-
INFO 2026-04-05 08:26:15,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 60 |
-
INFO 2026-04-05 08:29:07,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 61 |
-
INFO 2026-04-05 08:31:59,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 62 |
-
INFO 2026-04-05 08:34:50,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 63 |
-
INFO 2026-04-05 08:37:43,456 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 64 |
-
INFO 2026-04-05 08:40:35,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 65 |
-
INFO 2026-04-05 08:43:28,467 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 66 |
-
INFO 2026-04-05 08:46:21,132 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 67 |
-
INFO 2026-04-05 08:49:13,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 68 |
-
INFO 2026-04-05 08:52:04,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 69 |
-
INFO 2026-04-05 08:54:56,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 70 |
-
INFO 2026-04-05 08:57:48,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 71 |
-
INFO 2026-04-05 09:00:40,238 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 72 |
-
INFO 2026-04-05 09:03:32,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 73 |
-
INFO 2026-04-05 09:06:23,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 74 |
-
INFO 2026-04-05 09:09:15,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 75 |
-
INFO 2026-04-05 09:12:08,117 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 76 |
-
INFO 2026-04-05 09:14:59,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 77 |
-
INFO 2026-04-05 09:17:51,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 78 |
-
INFO 2026-04-05 09:20:42,298 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 79 |
-
INFO 2026-04-05 09:23:34,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 80 |
-
INFO 2026-04-05 09:26:26,063 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 81 |
-
INFO 2026-04-05 09:29:18,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 82 |
-
INFO 2026-04-05 09:32:09,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 83 |
-
INFO 2026-04-05 09:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 84 |
-
INFO 2026-04-05 09:37:52,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 85 |
-
INFO 2026-04-05 09:40:43,428 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 86 |
-
INFO 2026-04-05 09:43:34,586 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 87 |
-
INFO 2026-04-05 09:46:26,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 88 |
-
INFO 2026-04-05 09:49:17,267 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 89 |
-
INFO 2026-04-05 09:52:06,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 90 |
-
INFO 2026-04-05 09:54:57,889 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 91 |
-
INFO 2026-04-05 09:57:48,894 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 92 |
-
INFO 2026-04-05 10:00:41,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 93 |
-
INFO 2026-04-05 10:03:33,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 94 |
-
INFO 2026-04-05 10:06:24,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 95 |
-
INFO 2026-04-05 10:09:17,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 96 |
-
INFO 2026-04-05 10:12:18,806 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 97 |
-
INFO 2026-04-05 10:15:33,483 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 98 |
-
INFO 2026-04-05 10:18:24,744 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 99 |
-
INFO 2026-04-05 10:21:17,302 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 100 |
-
INFO 2026-04-05 10:24:08,917 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 101 |
-
INFO 2026-04-05 10:27:00,005 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 102 |
-
INFO 2026-04-05 10:29:51,959 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 103 |
-
INFO 2026-04-05 10:32:42,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 104 |
-
INFO 2026-04-05 10:35:34,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 105 |
-
INFO 2026-04-05 10:38:25,583 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 106 |
-
INFO 2026-04-05 10:41:18,350 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 107 |
-
INFO 2026-04-05 10:44:08,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 108 |
-
INFO 2026-04-05 10:47:00,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 109 |
-
INFO 2026-04-05 10:49:51,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 110 |
-
INFO 2026-04-05 10:52:43,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 111 |
-
INFO 2026-04-05 10:55:35,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 112 |
-
INFO 2026-04-05 10:58:26,949 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 113 |
-
INFO 2026-04-05 11:01:18,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 114 |
-
INFO 2026-04-05 11:04:10,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 115 |
-
INFO 2026-04-05 11:07:02,461 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 116 |
-
INFO 2026-04-05 11:09:53,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 117 |
-
INFO 2026-04-05 11:12:45,451 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 118 |
-
INFO 2026-04-05 11:15:36,492 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 119 |
-
INFO 2026-04-05 11:18:28,400 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 120 |
-
INFO 2026-04-05 11:21:20,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 121 |
-
INFO 2026-04-05 11:24:11,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 122 |
-
INFO 2026-04-05 11:27:02,198 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 123 |
-
INFO 2026-04-05 11:29:53,736 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 124 |
-
INFO 2026-04-05 11:32:44,011 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 125 |
-
INFO 2026-04-05 11:35:35,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 126 |
-
INFO 2026-04-05 11:38:25,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 127 |
-
INFO 2026-04-05 11:41:15,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 128 |
-
INFO 2026-04-05 11:44:05,134 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 129 |
-
INFO 2026-04-05 11:46:56,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 130 |
-
INFO 2026-04-05 11:49:47,469 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 131 |
-
INFO 2026-04-05 11:52:38,563 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 132 |
-
INFO 2026-04-05 11:55:30,289 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 133 |
-
INFO 2026-04-05 11:58:21,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 134 |
-
INFO 2026-04-05 12:01:13,424 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 135 |
-
INFO 2026-04-05 12:04:04,015 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 136 |
-
INFO 2026-04-05 12:06:55,196 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 137 |
-
INFO 2026-04-05 12:09:46,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 138 |
-
INFO 2026-04-05 12:12:37,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 139 |
-
INFO 2026-04-05 12:15:28,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 140 |
-
INFO 2026-04-05 12:18:19,903 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 141 |
-
INFO 2026-04-05 12:21:11,565 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 142 |
-
INFO 2026-04-05 12:24:02,399 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 143 |
-
INFO 2026-04-05 12:26:54,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 144 |
-
INFO 2026-04-05 12:29:45,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 145 |
-
INFO 2026-04-05 12:32:36,286 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 146 |
-
INFO 2026-04-05 12:35:36,955 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 147 |
-
INFO 2026-04-05 12:38:50,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 148 |
-
INFO 2026-04-05 12:41:41,343 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 149 |
-
INFO 2026-04-05 12:44:32,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 150 |
-
INFO 2026-04-05 12:47:23,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 151 |
-
INFO 2026-04-05 12:50:13,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 152 |
-
INFO 2026-04-05 12:53:06,274 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 153 |
-
INFO 2026-04-05 12:55:56,979 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 154 |
-
INFO 2026-04-05 12:58:48,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 155 |
-
INFO 2026-04-05 13:01:39,887 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 156 |
-
INFO 2026-04-05 13:04:31,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 157 |
-
INFO 2026-04-05 13:07:22,319 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 158 |
-
INFO 2026-04-05 13:10:15,398 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 159 |
-
INFO 2026-04-05 13:13:06,519 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 160 |
-
INFO 2026-04-05 13:15:57,164 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 161 |
-
INFO 2026-04-05 13:18:49,647 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 162 |
-
INFO 2026-04-05 13:21:40,162 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 163 |
-
INFO 2026-04-05 13:24:31,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 164 |
-
INFO 2026-04-05 13:27:22,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 165 |
-
INFO 2026-04-05 13:30:13,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 166 |
-
INFO 2026-04-05 13:33:06,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 167 |
-
INFO 2026-04-05 13:35:57,606 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 168 |
-
INFO 2026-04-05 13:38:47,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 169 |
-
INFO 2026-04-05 13:41:40,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 170 |
-
INFO 2026-04-05 13:44:31,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 171 |
-
INFO 2026-04-05 13:47:23,254 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 172 |
-
INFO 2026-04-05 13:50:14,660 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 173 |
-
INFO 2026-04-05 13:53:06,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 174 |
-
INFO 2026-04-05 13:55:58,107 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 175 |
-
INFO 2026-04-05 13:58:50,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 176 |
-
INFO 2026-04-05 14:01:41,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 177 |
-
INFO 2026-04-05 14:04:32,278 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 178 |
-
INFO 2026-04-05 14:07:24,300 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 179 |
-
INFO 2026-04-05 14:10:15,785 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 180 |
-
INFO 2026-04-05 14:13:07,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 181 |
-
INFO 2026-04-05 14:15:58,762 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 182 |
-
INFO 2026-04-05 14:18:49,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 183 |
-
INFO 2026-04-05 14:21:40,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 184 |
-
INFO 2026-04-05 14:24:31,803 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 185 |
-
INFO 2026-04-05 14:27:22,607 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 186 |
-
INFO 2026-04-05 14:30:14,379 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 187 |
-
INFO 2026-04-05 14:33:06,049 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 188 |
-
INFO 2026-04-05 14:35:56,094 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 189 |
-
INFO 2026-04-05 14:38:48,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 190 |
-
INFO 2026-04-05 14:41:40,352 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 191 |
-
INFO 2026-04-05 14:44:32,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 192 |
-
INFO 2026-04-05 14:47:25,215 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 193 |
-
INFO 2026-04-05 14:50:15,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 194 |
-
INFO 2026-04-05 14:53:07,347 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 195 |
-
INFO 2026-04-05 14:56:00,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 196 |
-
INFO 2026-04-05 14:59:01,670 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 197 |
-
INFO 2026-04-05 14:59:35,376 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=3
|
| 198 |
-
INFO 2026-04-06 07:33:53,166 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=3, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 199 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 200 |
-
hyperparams=ConfigDict[typing.Any](
|
| 201 |
-
lr=5e-05,
|
| 202 |
-
weight_decay=0.0,
|
| 203 |
-
),
|
| 204 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 205 |
-
torch_optimizer='torch.optim.AdamW',
|
| 206 |
-
), scheduler=SequentialSchedulerConfig(
|
| 207 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 208 |
-
milestones=ConfigList[int](
|
| 209 |
-
500,
|
| 210 |
-
),
|
| 211 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 212 |
-
warmup=TorchSchedulerConfig(
|
| 213 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 214 |
-
hyperparams=ConfigDict[typing.Any](
|
| 215 |
-
start_factor=0.0002,
|
| 216 |
-
end_factor=1.0,
|
| 217 |
-
total_iters=500,
|
| 218 |
-
),
|
| 219 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 220 |
-
),
|
| 221 |
-
const=TorchSchedulerConfig(
|
| 222 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 223 |
-
hyperparams=ConfigDict[typing.Any](
|
| 224 |
-
factor=1.0,
|
| 225 |
-
total_iters=300000,
|
| 226 |
-
),
|
| 227 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 228 |
-
),
|
| 229 |
-
),
|
| 230 |
-
)
|
| 231 |
-
INFO 2026-04-06 07:33:53,171 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 232 |
-
INFO 2026-04-06 07:33:56,071 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 233 |
-
INFO 2026-04-06 07:34:10,494 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 234 |
-
INFO 2026-04-06 07:34:18,352 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 235 |
-
INFO 2026-04-06 07:34:22,693 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 236 |
-
INFO 2026-04-06 07:34:22,694 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=26622.699999999997, dataset_name=bridge_paraphrase_k5
|
| 237 |
-
INFO 2026-04-06 07:34:22,735 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183231, rank=3
|
| 238 |
-
INFO 2026-04-06 07:34:23,620 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=3, status=start
|
| 239 |
-
INFO 2026-04-06 07:34:23,662 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
|
| 240 |
-
INFO 2026-04-06 07:34:23,664 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=30493.0
|
| 241 |
-
INFO 2026-04-06 07:34:23,671 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=3, status=start
|
| 242 |
-
INFO 2026-04-06 07:36:05,858 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=3, status=success
|
| 243 |
-
INFO 2026-04-06 07:36:27,877 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=3
|
| 244 |
-
INFO 2026-04-06 07:36:27,879 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 245 |
-
INFO 2026-04-06 07:36:27,940 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=2465170, parent_pid=2461888, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 246 |
-
INFO 2026-04-06 07:36:27,974 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=2465197, parent_pid=2461888, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 247 |
-
INFO 2026-04-06 07:39:43,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 248 |
-
INFO 2026-04-06 07:42:35,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 249 |
-
INFO 2026-04-06 07:45:26,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 250 |
-
INFO 2026-04-06 07:48:18,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 251 |
-
INFO 2026-04-06 07:51:08,588 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 252 |
-
INFO 2026-04-06 07:54:00,463 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 253 |
-
INFO 2026-04-06 07:56:51,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 254 |
-
INFO 2026-04-06 07:59:43,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 255 |
-
INFO 2026-04-06 08:02:34,656 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 256 |
-
INFO 2026-04-06 08:05:26,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 257 |
-
INFO 2026-04-06 08:08:18,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 258 |
-
INFO 2026-04-06 08:11:11,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 259 |
-
INFO 2026-04-06 08:14:03,148 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 260 |
-
INFO 2026-04-06 08:16:57,119 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 261 |
-
INFO 2026-04-06 08:19:48,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 262 |
-
INFO 2026-04-06 08:22:39,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 263 |
-
INFO 2026-04-06 08:25:31,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 264 |
-
INFO 2026-04-06 08:28:25,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 265 |
-
INFO 2026-04-06 08:31:16,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 266 |
-
INFO 2026-04-06 08:34:09,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 267 |
-
INFO 2026-04-06 08:37:01,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 268 |
-
INFO 2026-04-06 08:39:53,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 269 |
-
INFO 2026-04-06 08:42:45,381 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 270 |
-
INFO 2026-04-06 08:45:38,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 271 |
-
INFO 2026-04-06 08:48:30,988 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 272 |
-
INFO 2026-04-06 08:51:23,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 273 |
-
INFO 2026-04-06 08:54:13,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 274 |
-
INFO 2026-04-06 08:57:05,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 275 |
-
INFO 2026-04-06 08:59:57,948 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 276 |
-
INFO 2026-04-06 09:02:49,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 277 |
-
INFO 2026-04-06 09:05:41,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 278 |
-
INFO 2026-04-06 09:08:33,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 279 |
-
INFO 2026-04-06 09:11:25,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 280 |
-
INFO 2026-04-06 09:14:17,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 281 |
-
INFO 2026-04-06 09:17:08,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 282 |
-
INFO 2026-04-06 09:20:01,791 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 283 |
-
INFO 2026-04-06 09:22:53,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 284 |
-
INFO 2026-04-06 09:25:46,358 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 285 |
-
INFO 2026-04-06 09:28:40,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 286 |
-
INFO 2026-04-06 09:31:33,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 287 |
-
INFO 2026-04-06 09:34:24,731 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 288 |
-
INFO 2026-04-06 09:37:17,501 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 289 |
-
INFO 2026-04-06 09:40:08,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 290 |
-
INFO 2026-04-06 09:43:00,698 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 291 |
-
INFO 2026-04-06 09:45:52,659 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 292 |
-
INFO 2026-04-06 09:48:44,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 293 |
-
INFO 2026-04-06 09:51:36,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 294 |
-
INFO 2026-04-06 09:54:29,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 295 |
-
INFO 2026-04-06 09:57:20,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 296 |
-
INFO 2026-04-06 10:00:22,859 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 297 |
-
INFO 2026-04-06 10:03:48,882 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 298 |
-
INFO 2026-04-06 10:06:52,082 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 299 |
-
INFO 2026-04-06 10:09:59,025 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 300 |
-
INFO 2026-04-06 10:13:05,838 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 301 |
-
INFO 2026-04-06 10:16:11,931 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 302 |
-
INFO 2026-04-06 10:19:15,973 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 303 |
-
INFO 2026-04-06 10:22:24,662 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 304 |
-
INFO 2026-04-06 10:25:33,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 305 |
-
INFO 2026-04-06 10:28:42,169 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 306 |
-
INFO 2026-04-06 10:31:51,513 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 307 |
-
INFO 2026-04-06 10:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 308 |
-
INFO 2026-04-06 10:38:08,605 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 309 |
-
INFO 2026-04-06 10:41:15,592 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 310 |
-
INFO 2026-04-06 11:29:31,541 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=3, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 311 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 312 |
-
hyperparams=ConfigDict[typing.Any](
|
| 313 |
-
lr=5e-05,
|
| 314 |
-
weight_decay=0.0,
|
| 315 |
-
),
|
| 316 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 317 |
-
torch_optimizer='torch.optim.AdamW',
|
| 318 |
-
), scheduler=SequentialSchedulerConfig(
|
| 319 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 320 |
-
milestones=ConfigList[int](
|
| 321 |
-
500,
|
| 322 |
-
),
|
| 323 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 324 |
-
warmup=TorchSchedulerConfig(
|
| 325 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 326 |
-
hyperparams=ConfigDict[typing.Any](
|
| 327 |
-
start_factor=0.0002,
|
| 328 |
-
end_factor=1.0,
|
| 329 |
-
total_iters=500,
|
| 330 |
-
),
|
| 331 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 332 |
-
),
|
| 333 |
-
const=TorchSchedulerConfig(
|
| 334 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 335 |
-
hyperparams=ConfigDict[typing.Any](
|
| 336 |
-
factor=1.0,
|
| 337 |
-
total_iters=300000,
|
| 338 |
-
),
|
| 339 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 340 |
-
),
|
| 341 |
-
),
|
| 342 |
-
)
|
| 343 |
-
INFO 2026-04-06 11:29:31,542 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 344 |
-
INFO 2026-04-06 11:29:34,957 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 345 |
-
INFO 2026-04-06 11:29:43,734 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 346 |
-
INFO 2026-04-06 11:29:48,010 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 347 |
-
INFO 2026-04-06 11:29:51,747 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 348 |
-
INFO 2026-04-06 11:29:51,748 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=16790.199999999997, dataset_name=bridge_paraphrase_k5
|
| 349 |
-
INFO 2026-04-06 11:29:51,791 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183231, rank=3
|
| 350 |
-
INFO 2026-04-06 11:29:52,609 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=3, status=start
|
| 351 |
-
INFO 2026-04-06 11:29:52,648 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
|
| 352 |
-
INFO 2026-04-06 11:29:52,650 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=21107.399999999998
|
| 353 |
-
INFO 2026-04-06 11:29:52,655 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=3, status=start
|
| 354 |
-
INFO 2026-04-06 11:31:39,839 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=3, status=success
|
| 355 |
-
INFO 2026-04-06 11:32:01,842 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=3
|
| 356 |
-
INFO 2026-04-06 11:32:01,844 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 357 |
-
INFO 2026-04-06 11:32:02,012 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-7, worker_pid=3492781, parent_pid=3489443, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 358 |
-
INFO 2026-04-06 11:32:02,046 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-7, worker_pid=3492823, parent_pid=3489443, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 359 |
-
INFO 2026-04-06 11:35:29,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 360 |
-
INFO 2026-04-06 11:38:35,205 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 361 |
-
INFO 2026-04-06 11:41:41,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 362 |
-
INFO 2026-04-06 11:44:46,853 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 363 |
-
INFO 2026-04-06 11:47:52,316 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 364 |
-
INFO 2026-04-06 11:50:56,954 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 365 |
-
INFO 2026-04-06 11:54:03,028 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 366 |
-
INFO 2026-04-06 11:57:07,427 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 367 |
-
INFO 2026-04-06 12:00:13,670 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 368 |
-
INFO 2026-04-06 12:03:19,356 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 369 |
-
INFO 2026-04-06 12:06:23,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 370 |
-
INFO 2026-04-06 12:09:29,449 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 371 |
-
INFO 2026-04-06 12:12:35,573 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 372 |
-
INFO 2026-04-06 12:15:41,419 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 373 |
-
INFO 2026-04-06 12:18:46,707 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 374 |
-
INFO 2026-04-06 12:21:51,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 375 |
-
INFO 2026-04-06 12:24:57,751 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 376 |
-
INFO 2026-04-06 12:28:01,900 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 377 |
-
INFO 2026-04-06 12:31:07,226 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 378 |
-
INFO 2026-04-06 12:34:11,869 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 379 |
-
INFO 2026-04-06 12:37:16,742 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 380 |
-
INFO 2026-04-06 12:40:21,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 381 |
-
INFO 2026-04-06 12:43:25,439 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 382 |
-
INFO 2026-04-06 12:46:31,794 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 383 |
-
INFO 2026-04-06 12:49:37,509 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 384 |
-
INFO 2026-04-06 12:52:43,066 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 385 |
-
INFO 2026-04-06 12:55:47,773 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 386 |
-
INFO 2026-04-06 12:58:53,491 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 387 |
-
INFO 2026-04-06 13:01:58,054 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 388 |
-
INFO 2026-04-06 13:05:03,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 389 |
-
INFO 2026-04-06 13:08:09,161 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 390 |
-
INFO 2026-04-06 13:11:13,202 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 391 |
-
INFO 2026-04-06 13:14:19,371 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 392 |
-
INFO 2026-04-06 13:17:25,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 393 |
-
INFO 2026-04-06 13:20:30,205 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 394 |
-
INFO 2026-04-06 13:23:35,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 395 |
-
INFO 2026-04-06 13:26:41,883 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 396 |
-
INFO 2026-04-06 13:29:48,085 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 397 |
-
INFO 2026-04-06 13:32:52,621 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 398 |
-
INFO 2026-04-06 13:35:57,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 399 |
-
INFO 2026-04-06 13:39:03,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 400 |
-
INFO 2026-04-06 13:42:10,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 401 |
-
INFO 2026-04-06 13:45:14,642 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 402 |
-
INFO 2026-04-06 13:48:20,973 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 403 |
-
INFO 2026-04-06 13:51:25,870 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 404 |
-
INFO 2026-04-06 13:54:30,723 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 405 |
-
INFO 2026-04-06 13:57:36,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 406 |
-
INFO 2026-04-06 14:00:41,878 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 407 |
-
INFO 2026-04-06 14:03:49,035 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 408 |
-
INFO 2026-04-06 14:07:04,756 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 409 |
-
INFO 2026-04-06 14:10:39,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 410 |
-
INFO 2026-04-06 14:13:44,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 411 |
-
INFO 2026-04-06 14:16:49,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 412 |
-
INFO 2026-04-06 14:19:53,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 413 |
-
INFO 2026-04-06 14:22:58,780 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 414 |
-
INFO 2026-04-06 14:26:03,765 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 415 |
-
INFO 2026-04-06 14:29:07,820 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 416 |
-
INFO 2026-04-06 14:32:11,856 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 417 |
-
INFO 2026-04-06 14:35:16,311 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 418 |
-
INFO 2026-04-06 14:38:22,038 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 419 |
-
INFO 2026-04-06 14:41:25,477 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 420 |
-
INFO 2026-04-06 14:44:30,533 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 421 |
-
INFO 2026-04-06 14:47:36,055 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 422 |
-
INFO 2026-04-06 14:50:40,772 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 423 |
-
INFO 2026-04-06 14:53:45,038 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 424 |
-
INFO 2026-04-06 14:56:50,257 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 425 |
-
INFO 2026-04-06 14:59:56,690 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 426 |
-
INFO 2026-04-06 15:03:02,328 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 427 |
-
INFO 2026-04-07 00:41:05,434 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=3, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 428 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 429 |
-
hyperparams=ConfigDict[typing.Any](
|
| 430 |
-
lr=5e-05,
|
| 431 |
-
weight_decay=0.0,
|
| 432 |
-
),
|
| 433 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 434 |
-
torch_optimizer='torch.optim.AdamW',
|
| 435 |
-
), scheduler=SequentialSchedulerConfig(
|
| 436 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 437 |
-
milestones=ConfigList[int](
|
| 438 |
-
500,
|
| 439 |
-
),
|
| 440 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 441 |
-
warmup=TorchSchedulerConfig(
|
| 442 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 443 |
-
hyperparams=ConfigDict[typing.Any](
|
| 444 |
-
start_factor=0.0002,
|
| 445 |
-
end_factor=1.0,
|
| 446 |
-
total_iters=500,
|
| 447 |
-
),
|
| 448 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 449 |
-
),
|
| 450 |
-
const=TorchSchedulerConfig(
|
| 451 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 452 |
-
hyperparams=ConfigDict[typing.Any](
|
| 453 |
-
factor=1.0,
|
| 454 |
-
total_iters=300000,
|
| 455 |
-
),
|
| 456 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 457 |
-
),
|
| 458 |
-
),
|
| 459 |
-
)
|
| 460 |
-
INFO 2026-04-07 00:41:05,438 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 461 |
-
INFO 2026-04-07 00:41:08,565 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 462 |
-
INFO 2026-04-07 00:41:17,673 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 463 |
-
INFO 2026-04-07 00:41:22,588 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 464 |
-
INFO 2026-04-07 00:41:26,711 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 465 |
-
INFO 2026-04-07 00:41:26,711 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=18146.3, dataset_name=bridge_paraphrase_k5
|
| 466 |
-
INFO 2026-04-07 00:41:26,750 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183231, rank=3
|
| 467 |
-
INFO 2026-04-07 00:41:27,767 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=3, status=start
|
| 468 |
-
INFO 2026-04-07 00:41:27,806 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183231, num_data_pointers=160835, pct=87.78%, rank=3, status=success
|
| 469 |
-
INFO 2026-04-07 00:41:27,807 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=22369.5
|
| 470 |
-
INFO 2026-04-07 00:41:27,812 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=3, status=start
|
| 471 |
-
INFO 2026-04-07 00:43:48,102 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=3, status=success
|
| 472 |
-
INFO 2026-04-07 00:44:10,421 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=3
|
| 473 |
-
INFO 2026-04-07 00:44:10,423 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=3, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 474 |
-
INFO 2026-04-07 00:44:10,492 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=3313508, parent_pid=3310181, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 475 |
-
INFO 2026-04-07 00:44:10,499 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=3, hostname=msp3-5, worker_pid=3313515, parent_pid=3310181, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 476 |
-
INFO 2026-04-07 00:47:36,031 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 477 |
-
INFO 2026-04-07 00:50:38,868 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 478 |
-
INFO 2026-04-07 00:53:42,311 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 479 |
-
INFO 2026-04-07 00:56:45,085 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 480 |
-
INFO 2026-04-07 00:59:47,986 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 481 |
-
INFO 2026-04-07 01:02:50,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 482 |
-
INFO 2026-04-07 01:05:53,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 483 |
-
INFO 2026-04-07 01:08:56,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 484 |
-
INFO 2026-04-07 01:11:58,575 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 485 |
-
INFO 2026-04-07 01:15:02,202 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 486 |
-
INFO 2026-04-07 01:18:05,102 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 487 |
-
INFO 2026-04-07 01:21:08,473 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 488 |
-
INFO 2026-04-07 01:24:11,700 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 489 |
-
INFO 2026-04-07 01:27:15,144 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 490 |
-
INFO 2026-04-07 01:30:16,886 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 491 |
-
INFO 2026-04-07 01:33:19,435 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 492 |
-
INFO 2026-04-07 01:36:22,455 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 493 |
-
INFO 2026-04-07 01:39:25,144 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 494 |
-
INFO 2026-04-07 01:42:29,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 495 |
-
INFO 2026-04-07 01:45:32,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 496 |
-
INFO 2026-04-07 01:48:36,838 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 497 |
-
INFO 2026-04-07 01:51:39,490 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 498 |
-
INFO 2026-04-07 01:54:42,916 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 499 |
-
INFO 2026-04-07 01:57:46,410 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 500 |
-
INFO 2026-04-07 02:00:50,172 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 501 |
-
INFO 2026-04-07 02:03:54,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 502 |
-
INFO 2026-04-07 02:06:57,977 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 503 |
-
INFO 2026-04-07 02:10:02,504 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 504 |
-
INFO 2026-04-07 02:13:05,977 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 505 |
-
INFO 2026-04-07 02:16:10,152 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 506 |
-
INFO 2026-04-07 02:19:14,239 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 507 |
-
INFO 2026-04-07 02:22:17,138 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 508 |
-
INFO 2026-04-07 02:25:21,414 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 509 |
-
INFO 2026-04-07 02:28:24,737 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 510 |
-
INFO 2026-04-07 02:31:29,706 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 511 |
-
INFO 2026-04-07 02:34:33,435 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 512 |
-
INFO 2026-04-07 02:37:37,740 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 513 |
-
INFO 2026-04-07 02:40:41,389 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 514 |
-
INFO 2026-04-07 02:43:44,851 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 515 |
-
INFO 2026-04-07 02:46:48,385 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 516 |
-
INFO 2026-04-07 02:49:52,438 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 517 |
-
INFO 2026-04-07 02:52:56,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 518 |
-
INFO 2026-04-07 02:56:01,910 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 519 |
-
INFO 2026-04-07 02:59:07,494 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 520 |
-
INFO 2026-04-07 03:02:11,723 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 521 |
-
INFO 2026-04-07 03:05:16,729 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 522 |
-
INFO 2026-04-07 03:08:20,100 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 523 |
-
INFO 2026-04-07 03:11:23,981 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 524 |
-
INFO 2026-04-07 03:14:28,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 525 |
-
INFO 2026-04-07 03:17:51,598 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 526 |
-
INFO 2026-04-07 03:18:34,222 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=3
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e7576ddc2df7f608e2163d65ac4826d06e45769daa23ea34fe0212c932de2f58
|
| 3 |
+
size 64043
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_4.log
CHANGED
|
@@ -1,526 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
lr=5e-05,
|
| 5 |
-
weight_decay=0.0,
|
| 6 |
-
),
|
| 7 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 8 |
-
torch_optimizer='torch.optim.AdamW',
|
| 9 |
-
), scheduler=SequentialSchedulerConfig(
|
| 10 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 11 |
-
milestones=ConfigList[int](
|
| 12 |
-
500,
|
| 13 |
-
),
|
| 14 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 15 |
-
warmup=TorchSchedulerConfig(
|
| 16 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 17 |
-
hyperparams=ConfigDict[typing.Any](
|
| 18 |
-
start_factor=0.0002,
|
| 19 |
-
end_factor=1.0,
|
| 20 |
-
total_iters=500,
|
| 21 |
-
),
|
| 22 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 23 |
-
),
|
| 24 |
-
const=TorchSchedulerConfig(
|
| 25 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 26 |
-
hyperparams=ConfigDict[typing.Any](
|
| 27 |
-
factor=1.0,
|
| 28 |
-
total_iters=300000,
|
| 29 |
-
),
|
| 30 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 31 |
-
),
|
| 32 |
-
),
|
| 33 |
-
)
|
| 34 |
-
INFO 2026-04-05 07:47:39,880 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 35 |
-
INFO 2026-04-05 07:47:41,969 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge
|
| 36 |
-
INFO 2026-04-05 07:47:55,803 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 37 |
-
INFO 2026-04-05 07:48:00,005 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 38 |
-
INFO 2026-04-05 07:48:00,005 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=18036.0, dataset_name=bridge
|
| 39 |
-
INFO 2026-04-05 07:48:00,043 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183262, rank=4
|
| 40 |
-
INFO 2026-04-05 07:48:01,112 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge rank=4, status=start
|
| 41 |
-
INFO 2026-04-05 07:48:01,151 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
|
| 42 |
-
INFO 2026-04-05 07:48:01,152 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=21272.5
|
| 43 |
-
INFO 2026-04-05 07:48:34,458 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=4
|
| 44 |
-
INFO 2026-04-05 07:48:34,460 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 45 |
-
INFO 2026-04-05 07:48:34,520 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-2, worker_pid=3875751, parent_pid=3872893, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 46 |
-
INFO 2026-04-05 07:48:34,554 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-2, worker_pid=3875773, parent_pid=3872893, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 47 |
-
INFO 2026-04-05 07:51:52,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 48 |
-
INFO 2026-04-05 07:54:44,997 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 49 |
-
INFO 2026-04-05 07:57:35,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 50 |
-
INFO 2026-04-05 08:00:26,862 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 51 |
-
INFO 2026-04-05 08:03:18,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 52 |
-
INFO 2026-04-05 08:06:09,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 53 |
-
INFO 2026-04-05 08:09:01,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 54 |
-
INFO 2026-04-05 08:11:53,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 55 |
-
INFO 2026-04-05 08:14:44,578 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 56 |
-
INFO 2026-04-05 08:17:37,375 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 57 |
-
INFO 2026-04-05 08:20:30,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 58 |
-
INFO 2026-04-05 08:23:23,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 59 |
-
INFO 2026-04-05 08:26:15,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 60 |
-
INFO 2026-04-05 08:29:07,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 61 |
-
INFO 2026-04-05 08:31:59,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 62 |
-
INFO 2026-04-05 08:34:50,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 63 |
-
INFO 2026-04-05 08:37:43,456 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 64 |
-
INFO 2026-04-05 08:40:35,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 65 |
-
INFO 2026-04-05 08:43:28,467 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 66 |
-
INFO 2026-04-05 08:46:21,132 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 67 |
-
INFO 2026-04-05 08:49:13,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 68 |
-
INFO 2026-04-05 08:52:04,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 69 |
-
INFO 2026-04-05 08:54:56,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 70 |
-
INFO 2026-04-05 08:57:48,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 71 |
-
INFO 2026-04-05 09:00:40,238 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 72 |
-
INFO 2026-04-05 09:03:32,325 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 73 |
-
INFO 2026-04-05 09:06:23,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 74 |
-
INFO 2026-04-05 09:09:15,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 75 |
-
INFO 2026-04-05 09:12:08,117 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 76 |
-
INFO 2026-04-05 09:14:59,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 77 |
-
INFO 2026-04-05 09:17:51,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 78 |
-
INFO 2026-04-05 09:20:42,298 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 79 |
-
INFO 2026-04-05 09:23:34,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 80 |
-
INFO 2026-04-05 09:26:26,063 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 81 |
-
INFO 2026-04-05 09:29:18,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 82 |
-
INFO 2026-04-05 09:32:09,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 83 |
-
INFO 2026-04-05 09:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 84 |
-
INFO 2026-04-05 09:37:52,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 85 |
-
INFO 2026-04-05 09:40:43,428 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 86 |
-
INFO 2026-04-05 09:43:34,586 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 87 |
-
INFO 2026-04-05 09:46:26,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 88 |
-
INFO 2026-04-05 09:49:17,267 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 89 |
-
INFO 2026-04-05 09:52:06,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 90 |
-
INFO 2026-04-05 09:54:57,889 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 91 |
-
INFO 2026-04-05 09:57:48,894 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 92 |
-
INFO 2026-04-05 10:00:41,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 93 |
-
INFO 2026-04-05 10:03:33,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 94 |
-
INFO 2026-04-05 10:06:24,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 95 |
-
INFO 2026-04-05 10:09:17,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 96 |
-
INFO 2026-04-05 10:12:18,796 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 97 |
-
INFO 2026-04-05 10:15:33,483 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 98 |
-
INFO 2026-04-05 10:18:24,744 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 99 |
-
INFO 2026-04-05 10:21:17,302 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 100 |
-
INFO 2026-04-05 10:24:08,917 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 101 |
-
INFO 2026-04-05 10:27:00,005 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 102 |
-
INFO 2026-04-05 10:29:51,959 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 103 |
-
INFO 2026-04-05 10:32:42,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 104 |
-
INFO 2026-04-05 10:35:34,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 105 |
-
INFO 2026-04-05 10:38:25,583 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 106 |
-
INFO 2026-04-05 10:41:18,350 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 107 |
-
INFO 2026-04-05 10:44:08,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 108 |
-
INFO 2026-04-05 10:47:00,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 109 |
-
INFO 2026-04-05 10:49:51,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 110 |
-
INFO 2026-04-05 10:52:43,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 111 |
-
INFO 2026-04-05 10:55:35,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 112 |
-
INFO 2026-04-05 10:58:26,949 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 113 |
-
INFO 2026-04-05 11:01:18,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 114 |
-
INFO 2026-04-05 11:04:10,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 115 |
-
INFO 2026-04-05 11:07:02,461 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 116 |
-
INFO 2026-04-05 11:09:53,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 117 |
-
INFO 2026-04-05 11:12:45,451 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 118 |
-
INFO 2026-04-05 11:15:36,492 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 119 |
-
INFO 2026-04-05 11:18:28,400 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 120 |
-
INFO 2026-04-05 11:21:20,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 121 |
-
INFO 2026-04-05 11:24:11,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 122 |
-
INFO 2026-04-05 11:27:02,198 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 123 |
-
INFO 2026-04-05 11:29:53,736 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 124 |
-
INFO 2026-04-05 11:32:44,011 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 125 |
-
INFO 2026-04-05 11:35:35,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 126 |
-
INFO 2026-04-05 11:38:25,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 127 |
-
INFO 2026-04-05 11:41:15,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 128 |
-
INFO 2026-04-05 11:44:05,134 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 129 |
-
INFO 2026-04-05 11:46:56,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 130 |
-
INFO 2026-04-05 11:49:47,469 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 131 |
-
INFO 2026-04-05 11:52:38,563 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 132 |
-
INFO 2026-04-05 11:55:30,289 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 133 |
-
INFO 2026-04-05 11:58:21,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 134 |
-
INFO 2026-04-05 12:01:13,425 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 135 |
-
INFO 2026-04-05 12:04:04,015 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 136 |
-
INFO 2026-04-05 12:06:55,196 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 137 |
-
INFO 2026-04-05 12:09:46,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 138 |
-
INFO 2026-04-05 12:12:37,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 139 |
-
INFO 2026-04-05 12:15:28,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 140 |
-
INFO 2026-04-05 12:18:19,903 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 141 |
-
INFO 2026-04-05 12:21:11,565 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 142 |
-
INFO 2026-04-05 12:24:02,399 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 143 |
-
INFO 2026-04-05 12:26:54,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 144 |
-
INFO 2026-04-05 12:29:45,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 145 |
-
INFO 2026-04-05 12:32:36,286 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 146 |
-
INFO 2026-04-05 12:35:36,892 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 147 |
-
INFO 2026-04-05 12:38:50,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 148 |
-
INFO 2026-04-05 12:41:41,343 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 149 |
-
INFO 2026-04-05 12:44:32,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 150 |
-
INFO 2026-04-05 12:47:23,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 151 |
-
INFO 2026-04-05 12:50:13,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 152 |
-
INFO 2026-04-05 12:53:06,274 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 153 |
-
INFO 2026-04-05 12:55:56,979 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 154 |
-
INFO 2026-04-05 12:58:48,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 155 |
-
INFO 2026-04-05 13:01:39,887 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 156 |
-
INFO 2026-04-05 13:04:31,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 157 |
-
INFO 2026-04-05 13:07:22,318 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 158 |
-
INFO 2026-04-05 13:10:15,398 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 159 |
-
INFO 2026-04-05 13:13:06,519 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 160 |
-
INFO 2026-04-05 13:15:57,164 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 161 |
-
INFO 2026-04-05 13:18:49,647 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 162 |
-
INFO 2026-04-05 13:21:40,162 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 163 |
-
INFO 2026-04-05 13:24:31,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 164 |
-
INFO 2026-04-05 13:27:22,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 165 |
-
INFO 2026-04-05 13:30:13,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 166 |
-
INFO 2026-04-05 13:33:06,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 167 |
-
INFO 2026-04-05 13:35:57,606 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 168 |
-
INFO 2026-04-05 13:38:47,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 169 |
-
INFO 2026-04-05 13:41:40,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 170 |
-
INFO 2026-04-05 13:44:31,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 171 |
-
INFO 2026-04-05 13:47:23,254 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 172 |
-
INFO 2026-04-05 13:50:14,660 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 173 |
-
INFO 2026-04-05 13:53:06,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 174 |
-
INFO 2026-04-05 13:55:58,107 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 175 |
-
INFO 2026-04-05 13:58:50,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 176 |
-
INFO 2026-04-05 14:01:41,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 177 |
-
INFO 2026-04-05 14:04:32,278 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 178 |
-
INFO 2026-04-05 14:07:24,300 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 179 |
-
INFO 2026-04-05 14:10:15,785 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 180 |
-
INFO 2026-04-05 14:13:07,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 181 |
-
INFO 2026-04-05 14:15:58,762 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 182 |
-
INFO 2026-04-05 14:18:49,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 183 |
-
INFO 2026-04-05 14:21:40,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 184 |
-
INFO 2026-04-05 14:24:31,803 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 185 |
-
INFO 2026-04-05 14:27:22,607 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 186 |
-
INFO 2026-04-05 14:30:14,379 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 187 |
-
INFO 2026-04-05 14:33:06,049 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 188 |
-
INFO 2026-04-05 14:35:56,094 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 189 |
-
INFO 2026-04-05 14:38:48,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 190 |
-
INFO 2026-04-05 14:41:40,352 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 191 |
-
INFO 2026-04-05 14:44:32,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 192 |
-
INFO 2026-04-05 14:47:25,214 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 193 |
-
INFO 2026-04-05 14:50:15,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 194 |
-
INFO 2026-04-05 14:53:07,348 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 195 |
-
INFO 2026-04-05 14:56:00,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 196 |
-
INFO 2026-04-05 14:59:02,160 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 197 |
-
INFO 2026-04-05 14:59:35,376 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=4
|
| 198 |
-
INFO 2026-04-06 07:33:53,235 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=4, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 199 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 200 |
-
hyperparams=ConfigDict[typing.Any](
|
| 201 |
-
lr=5e-05,
|
| 202 |
-
weight_decay=0.0,
|
| 203 |
-
),
|
| 204 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 205 |
-
torch_optimizer='torch.optim.AdamW',
|
| 206 |
-
), scheduler=SequentialSchedulerConfig(
|
| 207 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 208 |
-
milestones=ConfigList[int](
|
| 209 |
-
500,
|
| 210 |
-
),
|
| 211 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 212 |
-
warmup=TorchSchedulerConfig(
|
| 213 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 214 |
-
hyperparams=ConfigDict[typing.Any](
|
| 215 |
-
start_factor=0.0002,
|
| 216 |
-
end_factor=1.0,
|
| 217 |
-
total_iters=500,
|
| 218 |
-
),
|
| 219 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 220 |
-
),
|
| 221 |
-
const=TorchSchedulerConfig(
|
| 222 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 223 |
-
hyperparams=ConfigDict[typing.Any](
|
| 224 |
-
factor=1.0,
|
| 225 |
-
total_iters=300000,
|
| 226 |
-
),
|
| 227 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 228 |
-
),
|
| 229 |
-
),
|
| 230 |
-
)
|
| 231 |
-
INFO 2026-04-06 07:33:53,239 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 232 |
-
INFO 2026-04-06 07:33:56,085 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 233 |
-
INFO 2026-04-06 07:34:05,279 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 234 |
-
INFO 2026-04-06 07:34:10,089 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 235 |
-
INFO 2026-04-06 07:34:14,165 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 236 |
-
INFO 2026-04-06 07:34:14,165 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=18080.2, dataset_name=bridge_paraphrase_k5
|
| 237 |
-
INFO 2026-04-06 07:34:14,205 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183262, rank=4
|
| 238 |
-
INFO 2026-04-06 07:34:15,165 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=4, status=start
|
| 239 |
-
INFO 2026-04-06 07:34:15,205 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
|
| 240 |
-
INFO 2026-04-06 07:34:15,206 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=21967.600000000002
|
| 241 |
-
INFO 2026-04-06 07:34:15,212 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=4, status=start
|
| 242 |
-
INFO 2026-04-06 07:36:05,814 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=4, status=success
|
| 243 |
-
INFO 2026-04-06 07:36:27,876 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=4
|
| 244 |
-
INFO 2026-04-06 07:36:27,879 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 245 |
-
INFO 2026-04-06 07:36:27,923 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=2465167, parent_pid=2461889, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 246 |
-
INFO 2026-04-06 07:36:27,947 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=2465176, parent_pid=2461889, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 247 |
-
INFO 2026-04-06 07:39:43,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 248 |
-
INFO 2026-04-06 07:42:35,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 249 |
-
INFO 2026-04-06 07:45:26,710 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 250 |
-
INFO 2026-04-06 07:48:18,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 251 |
-
INFO 2026-04-06 07:51:08,587 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 252 |
-
INFO 2026-04-06 07:54:00,463 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 253 |
-
INFO 2026-04-06 07:56:51,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 254 |
-
INFO 2026-04-06 07:59:43,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 255 |
-
INFO 2026-04-06 08:02:34,655 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 256 |
-
INFO 2026-04-06 08:05:26,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 257 |
-
INFO 2026-04-06 08:08:18,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 258 |
-
INFO 2026-04-06 08:11:11,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 259 |
-
INFO 2026-04-06 08:14:03,148 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 260 |
-
INFO 2026-04-06 08:16:57,119 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 261 |
-
INFO 2026-04-06 08:19:48,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 262 |
-
INFO 2026-04-06 08:22:39,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 263 |
-
INFO 2026-04-06 08:25:31,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 264 |
-
INFO 2026-04-06 08:28:25,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 265 |
-
INFO 2026-04-06 08:31:16,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 266 |
-
INFO 2026-04-06 08:34:09,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 267 |
-
INFO 2026-04-06 08:37:01,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 268 |
-
INFO 2026-04-06 08:39:53,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 269 |
-
INFO 2026-04-06 08:42:45,381 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 270 |
-
INFO 2026-04-06 08:45:38,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 271 |
-
INFO 2026-04-06 08:48:30,988 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 272 |
-
INFO 2026-04-06 08:51:23,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 273 |
-
INFO 2026-04-06 08:54:13,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 274 |
-
INFO 2026-04-06 08:57:05,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 275 |
-
INFO 2026-04-06 08:59:57,948 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 276 |
-
INFO 2026-04-06 09:02:49,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 277 |
-
INFO 2026-04-06 09:05:41,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 278 |
-
INFO 2026-04-06 09:08:33,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 279 |
-
INFO 2026-04-06 09:11:25,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 280 |
-
INFO 2026-04-06 09:14:17,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 281 |
-
INFO 2026-04-06 09:17:08,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 282 |
-
INFO 2026-04-06 09:20:01,791 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 283 |
-
INFO 2026-04-06 09:22:53,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 284 |
-
INFO 2026-04-06 09:25:46,358 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 285 |
-
INFO 2026-04-06 09:28:40,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 286 |
-
INFO 2026-04-06 09:31:33,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 287 |
-
INFO 2026-04-06 09:34:24,731 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 288 |
-
INFO 2026-04-06 09:37:17,501 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 289 |
-
INFO 2026-04-06 09:40:08,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 290 |
-
INFO 2026-04-06 09:43:00,698 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 291 |
-
INFO 2026-04-06 09:45:52,659 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 292 |
-
INFO 2026-04-06 09:48:44,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 293 |
-
INFO 2026-04-06 09:51:36,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 294 |
-
INFO 2026-04-06 09:54:29,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 295 |
-
INFO 2026-04-06 09:57:20,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 296 |
-
INFO 2026-04-06 10:00:22,725 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 297 |
-
INFO 2026-04-06 10:03:48,884 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 298 |
-
INFO 2026-04-06 10:06:52,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 299 |
-
INFO 2026-04-06 10:09:59,025 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 300 |
-
INFO 2026-04-06 10:13:05,840 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 301 |
-
INFO 2026-04-06 10:16:11,933 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 302 |
-
INFO 2026-04-06 10:19:15,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 303 |
-
INFO 2026-04-06 10:22:24,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 304 |
-
INFO 2026-04-06 10:25:33,536 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 305 |
-
INFO 2026-04-06 10:28:42,166 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 306 |
-
INFO 2026-04-06 10:31:51,506 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 307 |
-
INFO 2026-04-06 10:35:00,438 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 308 |
-
INFO 2026-04-06 10:38:08,601 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 309 |
-
INFO 2026-04-06 10:41:15,593 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 310 |
-
INFO 2026-04-06 11:29:31,565 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=4, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 311 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 312 |
-
hyperparams=ConfigDict[typing.Any](
|
| 313 |
-
lr=5e-05,
|
| 314 |
-
weight_decay=0.0,
|
| 315 |
-
),
|
| 316 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 317 |
-
torch_optimizer='torch.optim.AdamW',
|
| 318 |
-
), scheduler=SequentialSchedulerConfig(
|
| 319 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 320 |
-
milestones=ConfigList[int](
|
| 321 |
-
500,
|
| 322 |
-
),
|
| 323 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 324 |
-
warmup=TorchSchedulerConfig(
|
| 325 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 326 |
-
hyperparams=ConfigDict[typing.Any](
|
| 327 |
-
start_factor=0.0002,
|
| 328 |
-
end_factor=1.0,
|
| 329 |
-
total_iters=500,
|
| 330 |
-
),
|
| 331 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 332 |
-
),
|
| 333 |
-
const=TorchSchedulerConfig(
|
| 334 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 335 |
-
hyperparams=ConfigDict[typing.Any](
|
| 336 |
-
factor=1.0,
|
| 337 |
-
total_iters=300000,
|
| 338 |
-
),
|
| 339 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 340 |
-
),
|
| 341 |
-
),
|
| 342 |
-
)
|
| 343 |
-
INFO 2026-04-06 11:29:31,569 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 344 |
-
INFO 2026-04-06 11:29:34,896 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 345 |
-
INFO 2026-04-06 11:29:43,630 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 346 |
-
INFO 2026-04-06 11:29:47,749 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 347 |
-
INFO 2026-04-06 11:29:51,261 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 348 |
-
INFO 2026-04-06 11:29:51,262 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=16366.199999999999, dataset_name=bridge_paraphrase_k5
|
| 349 |
-
INFO 2026-04-06 11:29:51,302 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183262, rank=4
|
| 350 |
-
INFO 2026-04-06 11:29:52,100 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=4, status=start
|
| 351 |
-
INFO 2026-04-06 11:29:52,139 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
|
| 352 |
-
INFO 2026-04-06 11:29:52,142 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=20573.3
|
| 353 |
-
INFO 2026-04-06 11:29:52,149 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=4, status=start
|
| 354 |
-
INFO 2026-04-06 11:31:39,841 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=4, status=success
|
| 355 |
-
INFO 2026-04-06 11:32:01,842 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=4
|
| 356 |
-
INFO 2026-04-06 11:32:01,844 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 357 |
-
INFO 2026-04-06 11:32:02,007 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-7, worker_pid=3492779, parent_pid=3489444, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 358 |
-
INFO 2026-04-06 11:32:02,034 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-7, worker_pid=3492797, parent_pid=3489444, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 359 |
-
INFO 2026-04-06 11:35:29,335 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 360 |
-
INFO 2026-04-06 11:38:35,205 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 361 |
-
INFO 2026-04-06 11:41:41,429 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 362 |
-
INFO 2026-04-06 11:44:46,850 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 363 |
-
INFO 2026-04-06 11:47:52,316 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 364 |
-
INFO 2026-04-06 11:50:56,955 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 365 |
-
INFO 2026-04-06 11:54:03,030 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 366 |
-
INFO 2026-04-06 11:57:07,425 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 367 |
-
INFO 2026-04-06 12:00:13,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 368 |
-
INFO 2026-04-06 12:03:19,354 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 369 |
-
INFO 2026-04-06 12:06:23,698 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 370 |
-
INFO 2026-04-06 12:09:29,449 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 371 |
-
INFO 2026-04-06 12:12:35,571 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 372 |
-
INFO 2026-04-06 12:15:41,421 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 373 |
-
INFO 2026-04-06 12:18:46,705 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 374 |
-
INFO 2026-04-06 12:21:51,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 375 |
-
INFO 2026-04-06 12:24:57,748 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 376 |
-
INFO 2026-04-06 12:28:01,900 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 377 |
-
INFO 2026-04-06 12:31:07,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 378 |
-
INFO 2026-04-06 12:34:11,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 379 |
-
INFO 2026-04-06 12:37:16,741 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 380 |
-
INFO 2026-04-06 12:40:21,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 381 |
-
INFO 2026-04-06 12:43:25,437 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 382 |
-
INFO 2026-04-06 12:46:31,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 383 |
-
INFO 2026-04-06 12:49:37,508 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 384 |
-
INFO 2026-04-06 12:52:43,064 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 385 |
-
INFO 2026-04-06 12:55:47,771 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 386 |
-
INFO 2026-04-06 12:58:53,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 387 |
-
INFO 2026-04-06 13:01:58,052 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 388 |
-
INFO 2026-04-06 13:05:03,713 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 389 |
-
INFO 2026-04-06 13:08:09,156 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 390 |
-
INFO 2026-04-06 13:11:13,197 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 391 |
-
INFO 2026-04-06 13:14:19,370 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 392 |
-
INFO 2026-04-06 13:17:25,221 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 393 |
-
INFO 2026-04-06 13:20:30,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 394 |
-
INFO 2026-04-06 13:23:35,856 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 395 |
-
INFO 2026-04-06 13:26:41,882 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 396 |
-
INFO 2026-04-06 13:29:48,082 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 397 |
-
INFO 2026-04-06 13:32:52,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 398 |
-
INFO 2026-04-06 13:35:57,630 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 399 |
-
INFO 2026-04-06 13:39:03,246 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 400 |
-
INFO 2026-04-06 13:42:10,407 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 401 |
-
INFO 2026-04-06 13:45:14,630 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 402 |
-
INFO 2026-04-06 13:48:20,969 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 403 |
-
INFO 2026-04-06 13:51:25,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 404 |
-
INFO 2026-04-06 13:54:30,723 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 405 |
-
INFO 2026-04-06 13:57:36,004 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 406 |
-
INFO 2026-04-06 14:00:41,875 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 407 |
-
INFO 2026-04-06 14:03:49,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 408 |
-
INFO 2026-04-06 14:07:04,677 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 409 |
-
INFO 2026-04-06 14:10:39,271 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 410 |
-
INFO 2026-04-06 14:13:44,081 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 411 |
-
INFO 2026-04-06 14:16:49,285 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 412 |
-
INFO 2026-04-06 14:19:53,530 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 413 |
-
INFO 2026-04-06 14:22:58,779 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 414 |
-
INFO 2026-04-06 14:26:03,764 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 415 |
-
INFO 2026-04-06 14:29:07,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 416 |
-
INFO 2026-04-06 14:32:11,857 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 417 |
-
INFO 2026-04-06 14:35:16,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 418 |
-
INFO 2026-04-06 14:38:22,035 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 419 |
-
INFO 2026-04-06 14:41:25,477 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 420 |
-
INFO 2026-04-06 14:44:30,531 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 421 |
-
INFO 2026-04-06 14:47:36,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 422 |
-
INFO 2026-04-06 14:50:40,768 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 423 |
-
INFO 2026-04-06 14:53:45,036 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 424 |
-
INFO 2026-04-06 14:56:50,255 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 425 |
-
INFO 2026-04-06 14:59:56,689 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 426 |
-
INFO 2026-04-06 15:03:02,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 427 |
-
INFO 2026-04-07 00:41:05,401 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=4, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 428 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 429 |
-
hyperparams=ConfigDict[typing.Any](
|
| 430 |
-
lr=5e-05,
|
| 431 |
-
weight_decay=0.0,
|
| 432 |
-
),
|
| 433 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 434 |
-
torch_optimizer='torch.optim.AdamW',
|
| 435 |
-
), scheduler=SequentialSchedulerConfig(
|
| 436 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 437 |
-
milestones=ConfigList[int](
|
| 438 |
-
500,
|
| 439 |
-
),
|
| 440 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 441 |
-
warmup=TorchSchedulerConfig(
|
| 442 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 443 |
-
hyperparams=ConfigDict[typing.Any](
|
| 444 |
-
start_factor=0.0002,
|
| 445 |
-
end_factor=1.0,
|
| 446 |
-
total_iters=500,
|
| 447 |
-
),
|
| 448 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 449 |
-
),
|
| 450 |
-
const=TorchSchedulerConfig(
|
| 451 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 452 |
-
hyperparams=ConfigDict[typing.Any](
|
| 453 |
-
factor=1.0,
|
| 454 |
-
total_iters=300000,
|
| 455 |
-
),
|
| 456 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 457 |
-
),
|
| 458 |
-
),
|
| 459 |
-
)
|
| 460 |
-
INFO 2026-04-07 00:41:05,404 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 461 |
-
INFO 2026-04-07 00:41:08,947 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 462 |
-
INFO 2026-04-07 00:41:26,093 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 463 |
-
INFO 2026-04-07 00:41:36,337 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 464 |
-
INFO 2026-04-07 00:41:40,717 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 465 |
-
INFO 2026-04-07 00:41:40,718 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=31770.600000000002, dataset_name=bridge_paraphrase_k5
|
| 466 |
-
INFO 2026-04-07 00:41:40,759 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183262, rank=4
|
| 467 |
-
INFO 2026-04-07 00:41:41,580 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=4, status=start
|
| 468 |
-
INFO 2026-04-07 00:41:41,621 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183262, num_data_pointers=164614, pct=89.82%, rank=4, status=success
|
| 469 |
-
INFO 2026-04-07 00:41:41,624 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=36220.5
|
| 470 |
-
INFO 2026-04-07 00:41:41,631 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=4, status=start
|
| 471 |
-
INFO 2026-04-07 00:43:48,421 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=4, status=success
|
| 472 |
-
INFO 2026-04-07 00:44:10,422 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=4
|
| 473 |
-
INFO 2026-04-07 00:44:10,423 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=4, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 474 |
-
INFO 2026-04-07 00:44:10,500 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=3313512, parent_pid=3310182, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 475 |
-
INFO 2026-04-07 00:44:10,529 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=4, hostname=msp3-5, worker_pid=3313543, parent_pid=3310182, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 476 |
-
INFO 2026-04-07 00:47:36,030 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 477 |
-
INFO 2026-04-07 00:50:38,869 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 478 |
-
INFO 2026-04-07 00:53:42,313 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 479 |
-
INFO 2026-04-07 00:56:45,084 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 480 |
-
INFO 2026-04-07 00:59:47,986 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 481 |
-
INFO 2026-04-07 01:02:50,207 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 482 |
-
INFO 2026-04-07 01:05:53,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 483 |
-
INFO 2026-04-07 01:08:56,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 484 |
-
INFO 2026-04-07 01:11:58,573 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 485 |
-
INFO 2026-04-07 01:15:02,204 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 486 |
-
INFO 2026-04-07 01:18:05,102 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 487 |
-
INFO 2026-04-07 01:21:08,471 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 488 |
-
INFO 2026-04-07 01:24:11,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 489 |
-
INFO 2026-04-07 01:27:15,142 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 490 |
-
INFO 2026-04-07 01:30:16,884 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 491 |
-
INFO 2026-04-07 01:33:19,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 492 |
-
INFO 2026-04-07 01:36:22,455 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 493 |
-
INFO 2026-04-07 01:39:25,143 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 494 |
-
INFO 2026-04-07 01:42:29,625 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 495 |
-
INFO 2026-04-07 01:45:32,643 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 496 |
-
INFO 2026-04-07 01:48:36,837 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 497 |
-
INFO 2026-04-07 01:51:39,505 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 498 |
-
INFO 2026-04-07 01:54:42,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 499 |
-
INFO 2026-04-07 01:57:46,408 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 500 |
-
INFO 2026-04-07 02:00:50,171 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 501 |
-
INFO 2026-04-07 02:03:54,280 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 502 |
-
INFO 2026-04-07 02:06:57,975 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 503 |
-
INFO 2026-04-07 02:10:02,505 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 504 |
-
INFO 2026-04-07 02:13:05,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 505 |
-
INFO 2026-04-07 02:16:10,143 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 506 |
-
INFO 2026-04-07 02:19:14,256 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 507 |
-
INFO 2026-04-07 02:22:17,144 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 508 |
-
INFO 2026-04-07 02:25:21,415 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 509 |
-
INFO 2026-04-07 02:28:24,737 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 510 |
-
INFO 2026-04-07 02:31:29,708 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 511 |
-
INFO 2026-04-07 02:34:33,435 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 512 |
-
INFO 2026-04-07 02:37:37,740 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 513 |
-
INFO 2026-04-07 02:40:41,389 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 514 |
-
INFO 2026-04-07 02:43:44,852 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 515 |
-
INFO 2026-04-07 02:46:48,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 516 |
-
INFO 2026-04-07 02:49:52,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 517 |
-
INFO 2026-04-07 02:52:56,832 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 518 |
-
INFO 2026-04-07 02:56:01,912 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 519 |
-
INFO 2026-04-07 02:59:07,336 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 520 |
-
INFO 2026-04-07 03:02:11,725 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 521 |
-
INFO 2026-04-07 03:05:16,728 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 522 |
-
INFO 2026-04-07 03:08:20,110 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 523 |
-
INFO 2026-04-07 03:11:23,978 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 524 |
-
INFO 2026-04-07 03:14:28,687 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 525 |
-
INFO 2026-04-07 03:17:51,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 526 |
-
INFO 2026-04-07 03:18:34,222 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=4
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:00983e38d20ab3f03bff50fd7d395420a06294051a5e90ce02b7e6ac04513f4f
|
| 3 |
+
size 64032
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_5.log
CHANGED
|
@@ -1,526 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
lr=5e-05,
|
| 5 |
-
weight_decay=0.0,
|
| 6 |
-
),
|
| 7 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 8 |
-
torch_optimizer='torch.optim.AdamW',
|
| 9 |
-
), scheduler=SequentialSchedulerConfig(
|
| 10 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 11 |
-
milestones=ConfigList[int](
|
| 12 |
-
500,
|
| 13 |
-
),
|
| 14 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 15 |
-
warmup=TorchSchedulerConfig(
|
| 16 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 17 |
-
hyperparams=ConfigDict[typing.Any](
|
| 18 |
-
start_factor=0.0002,
|
| 19 |
-
end_factor=1.0,
|
| 20 |
-
total_iters=500,
|
| 21 |
-
),
|
| 22 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 23 |
-
),
|
| 24 |
-
const=TorchSchedulerConfig(
|
| 25 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 26 |
-
hyperparams=ConfigDict[typing.Any](
|
| 27 |
-
factor=1.0,
|
| 28 |
-
total_iters=300000,
|
| 29 |
-
),
|
| 30 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 31 |
-
),
|
| 32 |
-
),
|
| 33 |
-
)
|
| 34 |
-
INFO 2026-04-05 07:47:39,931 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 35 |
-
INFO 2026-04-05 07:47:41,994 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge
|
| 36 |
-
INFO 2026-04-05 07:48:03,608 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 37 |
-
INFO 2026-04-05 07:48:07,878 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 38 |
-
INFO 2026-04-05 07:48:07,878 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=25884.3, dataset_name=bridge
|
| 39 |
-
INFO 2026-04-05 07:48:07,919 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183239, rank=5
|
| 40 |
-
INFO 2026-04-05 07:48:08,795 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge rank=5, status=start
|
| 41 |
-
INFO 2026-04-05 07:48:08,835 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
|
| 42 |
-
INFO 2026-04-05 07:48:08,837 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=28905.6
|
| 43 |
-
INFO 2026-04-05 07:48:34,458 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=5
|
| 44 |
-
INFO 2026-04-05 07:48:34,460 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 45 |
-
INFO 2026-04-05 07:48:34,514 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-2, worker_pid=3875747, parent_pid=3872894, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 46 |
-
INFO 2026-04-05 07:48:34,535 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-2, worker_pid=3875754, parent_pid=3872894, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 47 |
-
INFO 2026-04-05 07:51:52,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 48 |
-
INFO 2026-04-05 07:54:44,997 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 49 |
-
INFO 2026-04-05 07:57:35,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 50 |
-
INFO 2026-04-05 08:00:26,862 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 51 |
-
INFO 2026-04-05 08:03:18,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 52 |
-
INFO 2026-04-05 08:06:09,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 53 |
-
INFO 2026-04-05 08:09:01,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 54 |
-
INFO 2026-04-05 08:11:53,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 55 |
-
INFO 2026-04-05 08:14:44,578 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 56 |
-
INFO 2026-04-05 08:17:37,375 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 57 |
-
INFO 2026-04-05 08:20:30,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 58 |
-
INFO 2026-04-05 08:23:23,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 59 |
-
INFO 2026-04-05 08:26:15,038 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 60 |
-
INFO 2026-04-05 08:29:07,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 61 |
-
INFO 2026-04-05 08:31:59,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 62 |
-
INFO 2026-04-05 08:34:50,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 63 |
-
INFO 2026-04-05 08:37:43,456 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 64 |
-
INFO 2026-04-05 08:40:35,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 65 |
-
INFO 2026-04-05 08:43:28,467 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 66 |
-
INFO 2026-04-05 08:46:21,132 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 67 |
-
INFO 2026-04-05 08:49:13,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 68 |
-
INFO 2026-04-05 08:52:04,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 69 |
-
INFO 2026-04-05 08:54:56,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 70 |
-
INFO 2026-04-05 08:57:48,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 71 |
-
INFO 2026-04-05 09:00:40,238 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 72 |
-
INFO 2026-04-05 09:03:32,325 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 73 |
-
INFO 2026-04-05 09:06:23,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 74 |
-
INFO 2026-04-05 09:09:15,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 75 |
-
INFO 2026-04-05 09:12:08,116 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 76 |
-
INFO 2026-04-05 09:14:59,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 77 |
-
INFO 2026-04-05 09:17:51,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 78 |
-
INFO 2026-04-05 09:20:42,298 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 79 |
-
INFO 2026-04-05 09:23:34,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 80 |
-
INFO 2026-04-05 09:26:26,064 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 81 |
-
INFO 2026-04-05 09:29:18,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 82 |
-
INFO 2026-04-05 09:32:09,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 83 |
-
INFO 2026-04-05 09:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 84 |
-
INFO 2026-04-05 09:37:52,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 85 |
-
INFO 2026-04-05 09:40:43,428 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 86 |
-
INFO 2026-04-05 09:43:34,586 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 87 |
-
INFO 2026-04-05 09:46:26,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 88 |
-
INFO 2026-04-05 09:49:17,267 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 89 |
-
INFO 2026-04-05 09:52:06,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 90 |
-
INFO 2026-04-05 09:54:57,889 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 91 |
-
INFO 2026-04-05 09:57:48,894 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 92 |
-
INFO 2026-04-05 10:00:41,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 93 |
-
INFO 2026-04-05 10:03:33,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 94 |
-
INFO 2026-04-05 10:06:24,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 95 |
-
INFO 2026-04-05 10:09:17,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 96 |
-
INFO 2026-04-05 10:12:18,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 97 |
-
INFO 2026-04-05 10:15:33,483 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 98 |
-
INFO 2026-04-05 10:18:24,744 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 99 |
-
INFO 2026-04-05 10:21:17,302 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 100 |
-
INFO 2026-04-05 10:24:08,917 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 101 |
-
INFO 2026-04-05 10:27:00,005 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 102 |
-
INFO 2026-04-05 10:29:51,958 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 103 |
-
INFO 2026-04-05 10:32:42,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 104 |
-
INFO 2026-04-05 10:35:34,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 105 |
-
INFO 2026-04-05 10:38:25,583 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 106 |
-
INFO 2026-04-05 10:41:18,350 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 107 |
-
INFO 2026-04-05 10:44:08,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 108 |
-
INFO 2026-04-05 10:47:00,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 109 |
-
INFO 2026-04-05 10:49:51,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 110 |
-
INFO 2026-04-05 10:52:43,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 111 |
-
INFO 2026-04-05 10:55:35,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 112 |
-
INFO 2026-04-05 10:58:26,949 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 113 |
-
INFO 2026-04-05 11:01:18,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 114 |
-
INFO 2026-04-05 11:04:10,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 115 |
-
INFO 2026-04-05 11:07:02,461 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 116 |
-
INFO 2026-04-05 11:09:53,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 117 |
-
INFO 2026-04-05 11:12:45,451 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 118 |
-
INFO 2026-04-05 11:15:36,492 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 119 |
-
INFO 2026-04-05 11:18:28,400 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 120 |
-
INFO 2026-04-05 11:21:20,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 121 |
-
INFO 2026-04-05 11:24:11,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 122 |
-
INFO 2026-04-05 11:27:02,198 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 123 |
-
INFO 2026-04-05 11:29:53,736 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 124 |
-
INFO 2026-04-05 11:32:44,011 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 125 |
-
INFO 2026-04-05 11:35:35,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 126 |
-
INFO 2026-04-05 11:38:25,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 127 |
-
INFO 2026-04-05 11:41:15,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 128 |
-
INFO 2026-04-05 11:44:05,134 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 129 |
-
INFO 2026-04-05 11:46:56,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 130 |
-
INFO 2026-04-05 11:49:47,469 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 131 |
-
INFO 2026-04-05 11:52:38,563 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 132 |
-
INFO 2026-04-05 11:55:30,289 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 133 |
-
INFO 2026-04-05 11:58:21,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 134 |
-
INFO 2026-04-05 12:01:13,424 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 135 |
-
INFO 2026-04-05 12:04:04,015 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 136 |
-
INFO 2026-04-05 12:06:55,196 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 137 |
-
INFO 2026-04-05 12:09:46,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 138 |
-
INFO 2026-04-05 12:12:37,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 139 |
-
INFO 2026-04-05 12:15:28,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 140 |
-
INFO 2026-04-05 12:18:19,903 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 141 |
-
INFO 2026-04-05 12:21:11,565 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 142 |
-
INFO 2026-04-05 12:24:02,399 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 143 |
-
INFO 2026-04-05 12:26:54,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 144 |
-
INFO 2026-04-05 12:29:45,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 145 |
-
INFO 2026-04-05 12:32:36,286 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 146 |
-
INFO 2026-04-05 12:35:36,962 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 147 |
-
INFO 2026-04-05 12:38:50,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 148 |
-
INFO 2026-04-05 12:41:41,343 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 149 |
-
INFO 2026-04-05 12:44:32,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 150 |
-
INFO 2026-04-05 12:47:23,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 151 |
-
INFO 2026-04-05 12:50:13,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 152 |
-
INFO 2026-04-05 12:53:06,274 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 153 |
-
INFO 2026-04-05 12:55:56,979 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 154 |
-
INFO 2026-04-05 12:58:48,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 155 |
-
INFO 2026-04-05 13:01:39,887 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 156 |
-
INFO 2026-04-05 13:04:31,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 157 |
-
INFO 2026-04-05 13:07:22,319 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 158 |
-
INFO 2026-04-05 13:10:15,398 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 159 |
-
INFO 2026-04-05 13:13:06,519 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 160 |
-
INFO 2026-04-05 13:15:57,164 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 161 |
-
INFO 2026-04-05 13:18:49,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 162 |
-
INFO 2026-04-05 13:21:40,162 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 163 |
-
INFO 2026-04-05 13:24:31,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 164 |
-
INFO 2026-04-05 13:27:22,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 165 |
-
INFO 2026-04-05 13:30:13,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 166 |
-
INFO 2026-04-05 13:33:06,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 167 |
-
INFO 2026-04-05 13:35:57,606 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 168 |
-
INFO 2026-04-05 13:38:47,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 169 |
-
INFO 2026-04-05 13:41:40,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 170 |
-
INFO 2026-04-05 13:44:31,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 171 |
-
INFO 2026-04-05 13:47:23,254 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 172 |
-
INFO 2026-04-05 13:50:14,660 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 173 |
-
INFO 2026-04-05 13:53:06,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 174 |
-
INFO 2026-04-05 13:55:58,107 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 175 |
-
INFO 2026-04-05 13:58:50,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 176 |
-
INFO 2026-04-05 14:01:41,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 177 |
-
INFO 2026-04-05 14:04:32,278 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 178 |
-
INFO 2026-04-05 14:07:24,300 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 179 |
-
INFO 2026-04-05 14:10:15,785 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 180 |
-
INFO 2026-04-05 14:13:07,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 181 |
-
INFO 2026-04-05 14:15:58,762 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 182 |
-
INFO 2026-04-05 14:18:49,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 183 |
-
INFO 2026-04-05 14:21:40,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 184 |
-
INFO 2026-04-05 14:24:31,803 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 185 |
-
INFO 2026-04-05 14:27:22,607 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 186 |
-
INFO 2026-04-05 14:30:14,379 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 187 |
-
INFO 2026-04-05 14:33:06,049 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 188 |
-
INFO 2026-04-05 14:35:56,094 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 189 |
-
INFO 2026-04-05 14:38:48,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 190 |
-
INFO 2026-04-05 14:41:40,352 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 191 |
-
INFO 2026-04-05 14:44:32,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 192 |
-
INFO 2026-04-05 14:47:25,214 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 193 |
-
INFO 2026-04-05 14:50:15,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 194 |
-
INFO 2026-04-05 14:53:07,347 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 195 |
-
INFO 2026-04-05 14:56:00,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 196 |
-
INFO 2026-04-05 14:59:02,157 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 197 |
-
INFO 2026-04-05 14:59:35,376 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=5
|
| 198 |
-
INFO 2026-04-06 07:33:53,199 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=5, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 199 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 200 |
-
hyperparams=ConfigDict[typing.Any](
|
| 201 |
-
lr=5e-05,
|
| 202 |
-
weight_decay=0.0,
|
| 203 |
-
),
|
| 204 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 205 |
-
torch_optimizer='torch.optim.AdamW',
|
| 206 |
-
), scheduler=SequentialSchedulerConfig(
|
| 207 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 208 |
-
milestones=ConfigList[int](
|
| 209 |
-
500,
|
| 210 |
-
),
|
| 211 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 212 |
-
warmup=TorchSchedulerConfig(
|
| 213 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 214 |
-
hyperparams=ConfigDict[typing.Any](
|
| 215 |
-
start_factor=0.0002,
|
| 216 |
-
end_factor=1.0,
|
| 217 |
-
total_iters=500,
|
| 218 |
-
),
|
| 219 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 220 |
-
),
|
| 221 |
-
const=TorchSchedulerConfig(
|
| 222 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 223 |
-
hyperparams=ConfigDict[typing.Any](
|
| 224 |
-
factor=1.0,
|
| 225 |
-
total_iters=300000,
|
| 226 |
-
),
|
| 227 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 228 |
-
),
|
| 229 |
-
),
|
| 230 |
-
)
|
| 231 |
-
INFO 2026-04-06 07:33:53,202 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 232 |
-
INFO 2026-04-06 07:33:56,070 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 233 |
-
INFO 2026-04-06 07:34:05,170 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 234 |
-
INFO 2026-04-06 07:34:09,978 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 235 |
-
INFO 2026-04-06 07:34:13,971 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 236 |
-
INFO 2026-04-06 07:34:13,971 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=17900.7, dataset_name=bridge_paraphrase_k5
|
| 237 |
-
INFO 2026-04-06 07:34:14,010 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183239, rank=5
|
| 238 |
-
INFO 2026-04-06 07:34:14,982 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=5, status=start
|
| 239 |
-
INFO 2026-04-06 07:34:15,021 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
|
| 240 |
-
INFO 2026-04-06 07:34:15,022 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=21820.3
|
| 241 |
-
INFO 2026-04-06 07:34:15,027 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=5, status=start
|
| 242 |
-
INFO 2026-04-06 07:36:05,815 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=5, status=success
|
| 243 |
-
INFO 2026-04-06 07:36:27,876 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=5
|
| 244 |
-
INFO 2026-04-06 07:36:27,879 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 245 |
-
INFO 2026-04-06 07:36:27,923 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=2465166, parent_pid=2461890, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 246 |
-
INFO 2026-04-06 07:36:27,945 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=2465172, parent_pid=2461890, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 247 |
-
INFO 2026-04-06 07:39:43,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 248 |
-
INFO 2026-04-06 07:42:35,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 249 |
-
INFO 2026-04-06 07:45:26,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 250 |
-
INFO 2026-04-06 07:48:18,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 251 |
-
INFO 2026-04-06 07:51:08,588 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 252 |
-
INFO 2026-04-06 07:54:00,463 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 253 |
-
INFO 2026-04-06 07:56:51,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 254 |
-
INFO 2026-04-06 07:59:43,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 255 |
-
INFO 2026-04-06 08:02:34,656 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 256 |
-
INFO 2026-04-06 08:05:26,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 257 |
-
INFO 2026-04-06 08:08:18,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 258 |
-
INFO 2026-04-06 08:11:11,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 259 |
-
INFO 2026-04-06 08:14:03,148 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 260 |
-
INFO 2026-04-06 08:16:57,119 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 261 |
-
INFO 2026-04-06 08:19:48,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 262 |
-
INFO 2026-04-06 08:22:39,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 263 |
-
INFO 2026-04-06 08:25:31,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 264 |
-
INFO 2026-04-06 08:28:25,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 265 |
-
INFO 2026-04-06 08:31:16,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 266 |
-
INFO 2026-04-06 08:34:09,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 267 |
-
INFO 2026-04-06 08:37:01,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 268 |
-
INFO 2026-04-06 08:39:53,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 269 |
-
INFO 2026-04-06 08:42:45,381 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 270 |
-
INFO 2026-04-06 08:45:38,330 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 271 |
-
INFO 2026-04-06 08:48:30,989 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 272 |
-
INFO 2026-04-06 08:51:23,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 273 |
-
INFO 2026-04-06 08:54:13,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 274 |
-
INFO 2026-04-06 08:57:05,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 275 |
-
INFO 2026-04-06 08:59:57,948 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 276 |
-
INFO 2026-04-06 09:02:49,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 277 |
-
INFO 2026-04-06 09:05:41,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 278 |
-
INFO 2026-04-06 09:08:33,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 279 |
-
INFO 2026-04-06 09:11:25,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 280 |
-
INFO 2026-04-06 09:14:17,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 281 |
-
INFO 2026-04-06 09:17:08,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 282 |
-
INFO 2026-04-06 09:20:01,791 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 283 |
-
INFO 2026-04-06 09:22:53,270 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 284 |
-
INFO 2026-04-06 09:25:46,358 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 285 |
-
INFO 2026-04-06 09:28:40,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 286 |
-
INFO 2026-04-06 09:31:33,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 287 |
-
INFO 2026-04-06 09:34:24,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 288 |
-
INFO 2026-04-06 09:37:17,501 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 289 |
-
INFO 2026-04-06 09:40:08,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 290 |
-
INFO 2026-04-06 09:43:00,698 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 291 |
-
INFO 2026-04-06 09:45:52,659 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 292 |
-
INFO 2026-04-06 09:48:44,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 293 |
-
INFO 2026-04-06 09:51:36,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 294 |
-
INFO 2026-04-06 09:54:29,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 295 |
-
INFO 2026-04-06 09:57:20,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 296 |
-
INFO 2026-04-06 10:00:22,924 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 297 |
-
INFO 2026-04-06 10:03:48,881 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 298 |
-
INFO 2026-04-06 10:06:52,069 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 299 |
-
INFO 2026-04-06 10:09:59,021 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 300 |
-
INFO 2026-04-06 10:13:05,836 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 301 |
-
INFO 2026-04-06 10:16:11,928 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 302 |
-
INFO 2026-04-06 10:19:15,972 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 303 |
-
INFO 2026-04-06 10:22:24,650 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 304 |
-
INFO 2026-04-06 10:25:33,535 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 305 |
-
INFO 2026-04-06 10:28:42,167 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 306 |
-
INFO 2026-04-06 10:31:51,504 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 307 |
-
INFO 2026-04-06 10:35:00,432 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 308 |
-
INFO 2026-04-06 10:38:08,599 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 309 |
-
INFO 2026-04-06 10:41:15,591 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 310 |
-
INFO 2026-04-06 11:29:31,543 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=5, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 311 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 312 |
-
hyperparams=ConfigDict[typing.Any](
|
| 313 |
-
lr=5e-05,
|
| 314 |
-
weight_decay=0.0,
|
| 315 |
-
),
|
| 316 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 317 |
-
torch_optimizer='torch.optim.AdamW',
|
| 318 |
-
), scheduler=SequentialSchedulerConfig(
|
| 319 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 320 |
-
milestones=ConfigList[int](
|
| 321 |
-
500,
|
| 322 |
-
),
|
| 323 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 324 |
-
warmup=TorchSchedulerConfig(
|
| 325 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 326 |
-
hyperparams=ConfigDict[typing.Any](
|
| 327 |
-
start_factor=0.0002,
|
| 328 |
-
end_factor=1.0,
|
| 329 |
-
total_iters=500,
|
| 330 |
-
),
|
| 331 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 332 |
-
),
|
| 333 |
-
const=TorchSchedulerConfig(
|
| 334 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 335 |
-
hyperparams=ConfigDict[typing.Any](
|
| 336 |
-
factor=1.0,
|
| 337 |
-
total_iters=300000,
|
| 338 |
-
),
|
| 339 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 340 |
-
),
|
| 341 |
-
),
|
| 342 |
-
)
|
| 343 |
-
INFO 2026-04-06 11:29:31,546 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 344 |
-
INFO 2026-04-06 11:29:34,908 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 345 |
-
INFO 2026-04-06 11:29:43,404 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 346 |
-
INFO 2026-04-06 11:29:47,554 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 347 |
-
INFO 2026-04-06 11:29:51,073 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 348 |
-
INFO 2026-04-06 11:29:51,073 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=16165.0, dataset_name=bridge_paraphrase_k5
|
| 349 |
-
INFO 2026-04-06 11:29:51,112 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183239, rank=5
|
| 350 |
-
INFO 2026-04-06 11:29:51,908 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=5, status=start
|
| 351 |
-
INFO 2026-04-06 11:29:51,947 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
|
| 352 |
-
INFO 2026-04-06 11:29:51,949 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=20403.0
|
| 353 |
-
INFO 2026-04-06 11:29:51,956 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=5, status=start
|
| 354 |
-
INFO 2026-04-06 11:31:39,838 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=5, status=success
|
| 355 |
-
INFO 2026-04-06 11:32:01,842 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=5
|
| 356 |
-
INFO 2026-04-06 11:32:01,844 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 357 |
-
INFO 2026-04-06 11:32:02,002 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-7, worker_pid=3492775, parent_pid=3489445, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 358 |
-
INFO 2026-04-06 11:32:02,028 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-7, worker_pid=3492785, parent_pid=3489445, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 359 |
-
INFO 2026-04-06 11:35:29,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 360 |
-
INFO 2026-04-06 11:38:35,202 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 361 |
-
INFO 2026-04-06 11:41:41,427 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 362 |
-
INFO 2026-04-06 11:44:46,849 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 363 |
-
INFO 2026-04-06 11:47:52,317 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 364 |
-
INFO 2026-04-06 11:50:56,956 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 365 |
-
INFO 2026-04-06 11:54:03,028 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 366 |
-
INFO 2026-04-06 11:57:07,422 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 367 |
-
INFO 2026-04-06 12:00:13,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 368 |
-
INFO 2026-04-06 12:03:19,351 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 369 |
-
INFO 2026-04-06 12:06:23,696 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 370 |
-
INFO 2026-04-06 12:09:29,448 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 371 |
-
INFO 2026-04-06 12:12:35,570 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 372 |
-
INFO 2026-04-06 12:15:41,418 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 373 |
-
INFO 2026-04-06 12:18:46,706 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 374 |
-
INFO 2026-04-06 12:21:51,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 375 |
-
INFO 2026-04-06 12:24:57,745 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 376 |
-
INFO 2026-04-06 12:28:01,897 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 377 |
-
INFO 2026-04-06 12:31:07,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 378 |
-
INFO 2026-04-06 12:34:11,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 379 |
-
INFO 2026-04-06 12:37:16,739 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 380 |
-
INFO 2026-04-06 12:40:21,217 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 381 |
-
INFO 2026-04-06 12:43:25,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 382 |
-
INFO 2026-04-06 12:46:31,794 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 383 |
-
INFO 2026-04-06 12:49:37,506 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 384 |
-
INFO 2026-04-06 12:52:43,062 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 385 |
-
INFO 2026-04-06 12:55:47,772 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 386 |
-
INFO 2026-04-06 12:58:53,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 387 |
-
INFO 2026-04-06 13:01:58,050 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 388 |
-
INFO 2026-04-06 13:05:03,714 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 389 |
-
INFO 2026-04-06 13:08:09,158 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 390 |
-
INFO 2026-04-06 13:11:13,197 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 391 |
-
INFO 2026-04-06 13:14:19,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 392 |
-
INFO 2026-04-06 13:17:25,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 393 |
-
INFO 2026-04-06 13:20:30,201 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 394 |
-
INFO 2026-04-06 13:23:35,857 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 395 |
-
INFO 2026-04-06 13:26:41,883 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 396 |
-
INFO 2026-04-06 13:29:48,084 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 397 |
-
INFO 2026-04-06 13:32:52,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 398 |
-
INFO 2026-04-06 13:35:57,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 399 |
-
INFO 2026-04-06 13:39:03,081 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 400 |
-
INFO 2026-04-06 13:42:10,403 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 401 |
-
INFO 2026-04-06 13:45:14,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 402 |
-
INFO 2026-04-06 13:48:20,973 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 403 |
-
INFO 2026-04-06 13:51:25,864 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 404 |
-
INFO 2026-04-06 13:54:30,720 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 405 |
-
INFO 2026-04-06 13:57:36,003 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 406 |
-
INFO 2026-04-06 14:00:41,877 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 407 |
-
INFO 2026-04-06 14:03:49,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 408 |
-
INFO 2026-04-06 14:07:04,771 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 409 |
-
INFO 2026-04-06 14:10:39,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 410 |
-
INFO 2026-04-06 14:13:44,081 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 411 |
-
INFO 2026-04-06 14:16:49,284 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 412 |
-
INFO 2026-04-06 14:19:53,530 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 413 |
-
INFO 2026-04-06 14:22:58,783 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 414 |
-
INFO 2026-04-06 14:26:03,761 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 415 |
-
INFO 2026-04-06 14:29:07,809 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 416 |
-
INFO 2026-04-06 14:32:11,852 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 417 |
-
INFO 2026-04-06 14:35:16,304 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 418 |
-
INFO 2026-04-06 14:38:22,035 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 419 |
-
INFO 2026-04-06 14:41:25,476 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 420 |
-
INFO 2026-04-06 14:44:30,532 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 421 |
-
INFO 2026-04-06 14:47:36,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 422 |
-
INFO 2026-04-06 14:50:40,768 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 423 |
-
INFO 2026-04-06 14:53:45,052 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 424 |
-
INFO 2026-04-06 14:56:50,253 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 425 |
-
INFO 2026-04-06 14:59:56,687 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 426 |
-
INFO 2026-04-06 15:03:02,324 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 427 |
-
INFO 2026-04-07 00:41:05,434 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=5, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 428 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 429 |
-
hyperparams=ConfigDict[typing.Any](
|
| 430 |
-
lr=5e-05,
|
| 431 |
-
weight_decay=0.0,
|
| 432 |
-
),
|
| 433 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 434 |
-
torch_optimizer='torch.optim.AdamW',
|
| 435 |
-
), scheduler=SequentialSchedulerConfig(
|
| 436 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 437 |
-
milestones=ConfigList[int](
|
| 438 |
-
500,
|
| 439 |
-
),
|
| 440 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 441 |
-
warmup=TorchSchedulerConfig(
|
| 442 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 443 |
-
hyperparams=ConfigDict[typing.Any](
|
| 444 |
-
start_factor=0.0002,
|
| 445 |
-
end_factor=1.0,
|
| 446 |
-
total_iters=500,
|
| 447 |
-
),
|
| 448 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 449 |
-
),
|
| 450 |
-
const=TorchSchedulerConfig(
|
| 451 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 452 |
-
hyperparams=ConfigDict[typing.Any](
|
| 453 |
-
factor=1.0,
|
| 454 |
-
total_iters=300000,
|
| 455 |
-
),
|
| 456 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 457 |
-
),
|
| 458 |
-
),
|
| 459 |
-
)
|
| 460 |
-
INFO 2026-04-07 00:41:05,435 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 461 |
-
INFO 2026-04-07 00:41:08,948 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 462 |
-
INFO 2026-04-07 00:41:26,272 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 463 |
-
INFO 2026-04-07 00:41:36,471 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 464 |
-
INFO 2026-04-07 00:41:40,790 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 465 |
-
INFO 2026-04-07 00:41:40,791 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=31842.4, dataset_name=bridge_paraphrase_k5
|
| 466 |
-
INFO 2026-04-07 00:41:40,833 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183239, rank=5
|
| 467 |
-
INFO 2026-04-07 00:41:41,660 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=5, status=start
|
| 468 |
-
INFO 2026-04-07 00:41:41,700 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183239, num_data_pointers=165383, pct=90.26%, rank=5, status=success
|
| 469 |
-
INFO 2026-04-07 00:41:41,701 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=36265.700000000004
|
| 470 |
-
INFO 2026-04-07 00:41:41,706 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=5, status=start
|
| 471 |
-
INFO 2026-04-07 00:43:48,408 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=5, status=success
|
| 472 |
-
INFO 2026-04-07 00:44:10,422 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=5
|
| 473 |
-
INFO 2026-04-07 00:44:10,423 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=5, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 474 |
-
INFO 2026-04-07 00:44:10,495 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=3313513, parent_pid=3310183, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 475 |
-
INFO 2026-04-07 00:44:10,551 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=5, hostname=msp3-5, worker_pid=3313548, parent_pid=3310183, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 476 |
-
INFO 2026-04-07 00:47:36,030 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 477 |
-
INFO 2026-04-07 00:50:38,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 478 |
-
INFO 2026-04-07 00:53:42,314 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 479 |
-
INFO 2026-04-07 00:56:45,084 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 480 |
-
INFO 2026-04-07 00:59:47,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 481 |
-
INFO 2026-04-07 01:02:50,206 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 482 |
-
INFO 2026-04-07 01:05:53,527 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 483 |
-
INFO 2026-04-07 01:08:56,071 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 484 |
-
INFO 2026-04-07 01:11:58,572 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 485 |
-
INFO 2026-04-07 01:15:02,202 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 486 |
-
INFO 2026-04-07 01:18:05,101 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 487 |
-
INFO 2026-04-07 01:21:08,472 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 488 |
-
INFO 2026-04-07 01:24:11,700 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 489 |
-
INFO 2026-04-07 01:27:15,143 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 490 |
-
INFO 2026-04-07 01:30:16,886 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 491 |
-
INFO 2026-04-07 01:33:19,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 492 |
-
INFO 2026-04-07 01:36:22,454 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 493 |
-
INFO 2026-04-07 01:39:25,144 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 494 |
-
INFO 2026-04-07 01:42:29,625 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 495 |
-
INFO 2026-04-07 01:45:32,634 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 496 |
-
INFO 2026-04-07 01:48:36,836 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 497 |
-
INFO 2026-04-07 01:51:39,520 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 498 |
-
INFO 2026-04-07 01:54:42,912 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 499 |
-
INFO 2026-04-07 01:57:46,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 500 |
-
INFO 2026-04-07 02:00:50,170 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 501 |
-
INFO 2026-04-07 02:03:54,280 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 502 |
-
INFO 2026-04-07 02:06:57,975 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 503 |
-
INFO 2026-04-07 02:10:02,505 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 504 |
-
INFO 2026-04-07 02:13:05,977 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 505 |
-
INFO 2026-04-07 02:16:10,140 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 506 |
-
INFO 2026-04-07 02:19:14,248 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 507 |
-
INFO 2026-04-07 02:22:17,141 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 508 |
-
INFO 2026-04-07 02:25:21,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 509 |
-
INFO 2026-04-07 02:28:24,737 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 510 |
-
INFO 2026-04-07 02:31:29,707 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 511 |
-
INFO 2026-04-07 02:34:33,437 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 512 |
-
INFO 2026-04-07 02:37:37,742 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 513 |
-
INFO 2026-04-07 02:40:41,391 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 514 |
-
INFO 2026-04-07 02:43:44,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 515 |
-
INFO 2026-04-07 02:46:48,387 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 516 |
-
INFO 2026-04-07 02:49:52,443 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 517 |
-
INFO 2026-04-07 02:52:56,836 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 518 |
-
INFO 2026-04-07 02:56:01,912 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 519 |
-
INFO 2026-04-07 02:59:07,348 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 520 |
-
INFO 2026-04-07 03:02:11,728 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 521 |
-
INFO 2026-04-07 03:05:16,730 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 522 |
-
INFO 2026-04-07 03:08:20,102 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 523 |
-
INFO 2026-04-07 03:11:23,980 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 524 |
-
INFO 2026-04-07 03:14:28,690 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 525 |
-
INFO 2026-04-07 03:17:51,392 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 526 |
-
INFO 2026-04-07 03:18:34,222 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=5
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7c185088a0665e7c6ae315b56822047d42ed5c9450b199d13cb4ebed348ff088
|
| 3 |
+
size 64010
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_6.log
CHANGED
|
@@ -1,526 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
lr=5e-05,
|
| 5 |
-
weight_decay=0.0,
|
| 6 |
-
),
|
| 7 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 8 |
-
torch_optimizer='torch.optim.AdamW',
|
| 9 |
-
), scheduler=SequentialSchedulerConfig(
|
| 10 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 11 |
-
milestones=ConfigList[int](
|
| 12 |
-
500,
|
| 13 |
-
),
|
| 14 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 15 |
-
warmup=TorchSchedulerConfig(
|
| 16 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 17 |
-
hyperparams=ConfigDict[typing.Any](
|
| 18 |
-
start_factor=0.0002,
|
| 19 |
-
end_factor=1.0,
|
| 20 |
-
total_iters=500,
|
| 21 |
-
),
|
| 22 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 23 |
-
),
|
| 24 |
-
const=TorchSchedulerConfig(
|
| 25 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 26 |
-
hyperparams=ConfigDict[typing.Any](
|
| 27 |
-
factor=1.0,
|
| 28 |
-
total_iters=300000,
|
| 29 |
-
),
|
| 30 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 31 |
-
),
|
| 32 |
-
),
|
| 33 |
-
)
|
| 34 |
-
INFO 2026-04-05 07:47:39,858 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 35 |
-
INFO 2026-04-05 07:47:42,026 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge
|
| 36 |
-
INFO 2026-04-05 07:48:07,528 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 37 |
-
INFO 2026-04-05 07:48:11,830 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 38 |
-
INFO 2026-04-05 07:48:11,830 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=29804.1, dataset_name=bridge
|
| 39 |
-
INFO 2026-04-05 07:48:11,870 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183295, rank=6
|
| 40 |
-
INFO 2026-04-05 07:48:12,722 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge rank=6, status=start
|
| 41 |
-
INFO 2026-04-05 07:48:12,763 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
|
| 42 |
-
INFO 2026-04-05 07:48:12,764 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=32905.700000000004
|
| 43 |
-
INFO 2026-04-05 07:48:34,459 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=6
|
| 44 |
-
INFO 2026-04-05 07:48:34,460 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 45 |
-
INFO 2026-04-05 07:48:34,519 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-2, worker_pid=3875750, parent_pid=3872895, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 46 |
-
INFO 2026-04-05 07:48:34,546 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-2, worker_pid=3875764, parent_pid=3872895, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 47 |
-
INFO 2026-04-05 07:51:52,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 48 |
-
INFO 2026-04-05 07:54:44,997 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 49 |
-
INFO 2026-04-05 07:57:35,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 50 |
-
INFO 2026-04-05 08:00:26,862 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 51 |
-
INFO 2026-04-05 08:03:18,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 52 |
-
INFO 2026-04-05 08:06:09,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 53 |
-
INFO 2026-04-05 08:09:01,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 54 |
-
INFO 2026-04-05 08:11:53,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 55 |
-
INFO 2026-04-05 08:14:44,578 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 56 |
-
INFO 2026-04-05 08:17:37,375 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 57 |
-
INFO 2026-04-05 08:20:30,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 58 |
-
INFO 2026-04-05 08:23:23,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 59 |
-
INFO 2026-04-05 08:26:15,038 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 60 |
-
INFO 2026-04-05 08:29:07,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 61 |
-
INFO 2026-04-05 08:31:59,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 62 |
-
INFO 2026-04-05 08:34:50,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 63 |
-
INFO 2026-04-05 08:37:43,456 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 64 |
-
INFO 2026-04-05 08:40:35,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 65 |
-
INFO 2026-04-05 08:43:28,467 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 66 |
-
INFO 2026-04-05 08:46:21,132 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 67 |
-
INFO 2026-04-05 08:49:13,209 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 68 |
-
INFO 2026-04-05 08:52:04,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 69 |
-
INFO 2026-04-05 08:54:56,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 70 |
-
INFO 2026-04-05 08:57:48,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 71 |
-
INFO 2026-04-05 09:00:40,238 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 72 |
-
INFO 2026-04-05 09:03:32,325 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 73 |
-
INFO 2026-04-05 09:06:23,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 74 |
-
INFO 2026-04-05 09:09:15,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 75 |
-
INFO 2026-04-05 09:12:08,117 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 76 |
-
INFO 2026-04-05 09:14:59,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 77 |
-
INFO 2026-04-05 09:17:51,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 78 |
-
INFO 2026-04-05 09:20:42,298 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 79 |
-
INFO 2026-04-05 09:23:34,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 80 |
-
INFO 2026-04-05 09:26:26,064 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 81 |
-
INFO 2026-04-05 09:29:18,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 82 |
-
INFO 2026-04-05 09:32:09,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 83 |
-
INFO 2026-04-05 09:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 84 |
-
INFO 2026-04-05 09:37:52,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 85 |
-
INFO 2026-04-05 09:40:43,428 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 86 |
-
INFO 2026-04-05 09:43:34,586 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 87 |
-
INFO 2026-04-05 09:46:26,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 88 |
-
INFO 2026-04-05 09:49:17,267 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 89 |
-
INFO 2026-04-05 09:52:06,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 90 |
-
INFO 2026-04-05 09:54:57,889 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 91 |
-
INFO 2026-04-05 09:57:48,894 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 92 |
-
INFO 2026-04-05 10:00:41,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 93 |
-
INFO 2026-04-05 10:03:33,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 94 |
-
INFO 2026-04-05 10:06:24,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 95 |
-
INFO 2026-04-05 10:09:17,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 96 |
-
INFO 2026-04-05 10:12:18,802 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 97 |
-
INFO 2026-04-05 10:15:33,483 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 98 |
-
INFO 2026-04-05 10:18:24,744 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 99 |
-
INFO 2026-04-05 10:21:17,302 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 100 |
-
INFO 2026-04-05 10:24:08,917 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 101 |
-
INFO 2026-04-05 10:27:00,005 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 102 |
-
INFO 2026-04-05 10:29:51,959 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 103 |
-
INFO 2026-04-05 10:32:42,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 104 |
-
INFO 2026-04-05 10:35:34,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 105 |
-
INFO 2026-04-05 10:38:25,583 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 106 |
-
INFO 2026-04-05 10:41:18,350 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 107 |
-
INFO 2026-04-05 10:44:08,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 108 |
-
INFO 2026-04-05 10:47:00,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 109 |
-
INFO 2026-04-05 10:49:51,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 110 |
-
INFO 2026-04-05 10:52:43,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 111 |
-
INFO 2026-04-05 10:55:35,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 112 |
-
INFO 2026-04-05 10:58:26,949 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 113 |
-
INFO 2026-04-05 11:01:18,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 114 |
-
INFO 2026-04-05 11:04:10,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 115 |
-
INFO 2026-04-05 11:07:02,461 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 116 |
-
INFO 2026-04-05 11:09:53,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 117 |
-
INFO 2026-04-05 11:12:45,451 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 118 |
-
INFO 2026-04-05 11:15:36,492 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 119 |
-
INFO 2026-04-05 11:18:28,400 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 120 |
-
INFO 2026-04-05 11:21:20,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 121 |
-
INFO 2026-04-05 11:24:11,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 122 |
-
INFO 2026-04-05 11:27:02,198 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 123 |
-
INFO 2026-04-05 11:29:53,736 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 124 |
-
INFO 2026-04-05 11:32:44,011 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 125 |
-
INFO 2026-04-05 11:35:35,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 126 |
-
INFO 2026-04-05 11:38:25,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 127 |
-
INFO 2026-04-05 11:41:15,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 128 |
-
INFO 2026-04-05 11:44:05,134 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 129 |
-
INFO 2026-04-05 11:46:56,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 130 |
-
INFO 2026-04-05 11:49:47,469 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 131 |
-
INFO 2026-04-05 11:52:38,563 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 132 |
-
INFO 2026-04-05 11:55:30,290 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 133 |
-
INFO 2026-04-05 11:58:21,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 134 |
-
INFO 2026-04-05 12:01:13,424 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 135 |
-
INFO 2026-04-05 12:04:04,015 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 136 |
-
INFO 2026-04-05 12:06:55,195 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 137 |
-
INFO 2026-04-05 12:09:46,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 138 |
-
INFO 2026-04-05 12:12:37,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 139 |
-
INFO 2026-04-05 12:15:28,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 140 |
-
INFO 2026-04-05 12:18:19,903 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 141 |
-
INFO 2026-04-05 12:21:11,565 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 142 |
-
INFO 2026-04-05 12:24:02,399 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 143 |
-
INFO 2026-04-05 12:26:54,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 144 |
-
INFO 2026-04-05 12:29:45,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 145 |
-
INFO 2026-04-05 12:32:36,286 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 146 |
-
INFO 2026-04-05 12:35:36,928 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 147 |
-
INFO 2026-04-05 12:38:50,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 148 |
-
INFO 2026-04-05 12:41:41,343 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 149 |
-
INFO 2026-04-05 12:44:32,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 150 |
-
INFO 2026-04-05 12:47:23,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 151 |
-
INFO 2026-04-05 12:50:13,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 152 |
-
INFO 2026-04-05 12:53:06,274 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 153 |
-
INFO 2026-04-05 12:55:56,979 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 154 |
-
INFO 2026-04-05 12:58:48,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 155 |
-
INFO 2026-04-05 13:01:39,887 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 156 |
-
INFO 2026-04-05 13:04:31,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 157 |
-
INFO 2026-04-05 13:07:22,319 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 158 |
-
INFO 2026-04-05 13:10:15,398 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 159 |
-
INFO 2026-04-05 13:13:06,519 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 160 |
-
INFO 2026-04-05 13:15:57,164 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 161 |
-
INFO 2026-04-05 13:18:49,647 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 162 |
-
INFO 2026-04-05 13:21:40,162 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 163 |
-
INFO 2026-04-05 13:24:31,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 164 |
-
INFO 2026-04-05 13:27:22,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 165 |
-
INFO 2026-04-05 13:30:13,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 166 |
-
INFO 2026-04-05 13:33:06,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 167 |
-
INFO 2026-04-05 13:35:57,606 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 168 |
-
INFO 2026-04-05 13:38:47,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 169 |
-
INFO 2026-04-05 13:41:40,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 170 |
-
INFO 2026-04-05 13:44:31,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 171 |
-
INFO 2026-04-05 13:47:23,254 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 172 |
-
INFO 2026-04-05 13:50:14,660 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 173 |
-
INFO 2026-04-05 13:53:06,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 174 |
-
INFO 2026-04-05 13:55:58,107 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 175 |
-
INFO 2026-04-05 13:58:50,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 176 |
-
INFO 2026-04-05 14:01:41,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 177 |
-
INFO 2026-04-05 14:04:32,278 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 178 |
-
INFO 2026-04-05 14:07:24,300 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 179 |
-
INFO 2026-04-05 14:10:15,786 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 180 |
-
INFO 2026-04-05 14:13:07,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 181 |
-
INFO 2026-04-05 14:15:58,762 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 182 |
-
INFO 2026-04-05 14:18:49,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 183 |
-
INFO 2026-04-05 14:21:40,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 184 |
-
INFO 2026-04-05 14:24:31,803 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 185 |
-
INFO 2026-04-05 14:27:22,607 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 186 |
-
INFO 2026-04-05 14:30:14,380 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 187 |
-
INFO 2026-04-05 14:33:06,049 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 188 |
-
INFO 2026-04-05 14:35:56,094 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 189 |
-
INFO 2026-04-05 14:38:48,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 190 |
-
INFO 2026-04-05 14:41:40,352 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 191 |
-
INFO 2026-04-05 14:44:32,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 192 |
-
INFO 2026-04-05 14:47:25,215 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 193 |
-
INFO 2026-04-05 14:50:15,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 194 |
-
INFO 2026-04-05 14:53:07,347 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 195 |
-
INFO 2026-04-05 14:56:00,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 196 |
-
INFO 2026-04-05 14:59:02,280 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 197 |
-
INFO 2026-04-05 14:59:35,377 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=6
|
| 198 |
-
INFO 2026-04-06 07:33:53,201 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=6, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 199 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 200 |
-
hyperparams=ConfigDict[typing.Any](
|
| 201 |
-
lr=5e-05,
|
| 202 |
-
weight_decay=0.0,
|
| 203 |
-
),
|
| 204 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 205 |
-
torch_optimizer='torch.optim.AdamW',
|
| 206 |
-
), scheduler=SequentialSchedulerConfig(
|
| 207 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 208 |
-
milestones=ConfigList[int](
|
| 209 |
-
500,
|
| 210 |
-
),
|
| 211 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 212 |
-
warmup=TorchSchedulerConfig(
|
| 213 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 214 |
-
hyperparams=ConfigDict[typing.Any](
|
| 215 |
-
start_factor=0.0002,
|
| 216 |
-
end_factor=1.0,
|
| 217 |
-
total_iters=500,
|
| 218 |
-
),
|
| 219 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 220 |
-
),
|
| 221 |
-
const=TorchSchedulerConfig(
|
| 222 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 223 |
-
hyperparams=ConfigDict[typing.Any](
|
| 224 |
-
factor=1.0,
|
| 225 |
-
total_iters=300000,
|
| 226 |
-
),
|
| 227 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 228 |
-
),
|
| 229 |
-
),
|
| 230 |
-
)
|
| 231 |
-
INFO 2026-04-06 07:33:53,203 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 232 |
-
INFO 2026-04-06 07:33:56,239 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 233 |
-
INFO 2026-04-06 07:34:13,865 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 234 |
-
INFO 2026-04-06 07:34:24,372 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 235 |
-
INFO 2026-04-06 07:34:28,401 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 236 |
-
INFO 2026-04-06 07:34:28,401 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=32162.199999999997, dataset_name=bridge_paraphrase_k5
|
| 237 |
-
INFO 2026-04-06 07:34:28,444 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183295, rank=6
|
| 238 |
-
INFO 2026-04-06 07:34:29,334 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=6, status=start
|
| 239 |
-
INFO 2026-04-06 07:34:29,376 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
|
| 240 |
-
INFO 2026-04-06 07:34:29,378 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=36174.799999999996
|
| 241 |
-
INFO 2026-04-06 07:34:29,385 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=6, status=start
|
| 242 |
-
INFO 2026-04-06 07:36:05,852 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=6, status=success
|
| 243 |
-
INFO 2026-04-06 07:36:27,877 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=6
|
| 244 |
-
INFO 2026-04-06 07:36:27,879 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 245 |
-
INFO 2026-04-06 07:36:27,950 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=2465175, parent_pid=2461891, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 246 |
-
INFO 2026-04-06 07:36:27,996 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=2465219, parent_pid=2461891, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 247 |
-
INFO 2026-04-06 07:39:43,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 248 |
-
INFO 2026-04-06 07:42:35,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 249 |
-
INFO 2026-04-06 07:45:26,710 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 250 |
-
INFO 2026-04-06 07:48:18,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 251 |
-
INFO 2026-04-06 07:51:08,588 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 252 |
-
INFO 2026-04-06 07:54:00,463 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 253 |
-
INFO 2026-04-06 07:56:51,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 254 |
-
INFO 2026-04-06 07:59:43,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 255 |
-
INFO 2026-04-06 08:02:34,655 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 256 |
-
INFO 2026-04-06 08:05:26,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 257 |
-
INFO 2026-04-06 08:08:18,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 258 |
-
INFO 2026-04-06 08:11:11,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 259 |
-
INFO 2026-04-06 08:14:03,148 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 260 |
-
INFO 2026-04-06 08:16:57,119 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 261 |
-
INFO 2026-04-06 08:19:48,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 262 |
-
INFO 2026-04-06 08:22:39,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 263 |
-
INFO 2026-04-06 08:25:31,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 264 |
-
INFO 2026-04-06 08:28:25,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 265 |
-
INFO 2026-04-06 08:31:16,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 266 |
-
INFO 2026-04-06 08:34:09,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 267 |
-
INFO 2026-04-06 08:37:01,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 268 |
-
INFO 2026-04-06 08:39:53,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 269 |
-
INFO 2026-04-06 08:42:45,381 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 270 |
-
INFO 2026-04-06 08:45:38,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 271 |
-
INFO 2026-04-06 08:48:30,988 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 272 |
-
INFO 2026-04-06 08:51:23,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 273 |
-
INFO 2026-04-06 08:54:13,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 274 |
-
INFO 2026-04-06 08:57:05,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 275 |
-
INFO 2026-04-06 08:59:57,948 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 276 |
-
INFO 2026-04-06 09:02:49,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 277 |
-
INFO 2026-04-06 09:05:41,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 278 |
-
INFO 2026-04-06 09:08:33,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 279 |
-
INFO 2026-04-06 09:11:25,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 280 |
-
INFO 2026-04-06 09:14:17,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 281 |
-
INFO 2026-04-06 09:17:08,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 282 |
-
INFO 2026-04-06 09:20:01,792 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 283 |
-
INFO 2026-04-06 09:22:53,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 284 |
-
INFO 2026-04-06 09:25:46,358 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 285 |
-
INFO 2026-04-06 09:28:40,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 286 |
-
INFO 2026-04-06 09:31:33,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 287 |
-
INFO 2026-04-06 09:34:24,731 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 288 |
-
INFO 2026-04-06 09:37:17,501 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 289 |
-
INFO 2026-04-06 09:40:08,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 290 |
-
INFO 2026-04-06 09:43:00,698 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 291 |
-
INFO 2026-04-06 09:45:52,659 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 292 |
-
INFO 2026-04-06 09:48:44,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 293 |
-
INFO 2026-04-06 09:51:36,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 294 |
-
INFO 2026-04-06 09:54:29,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 295 |
-
INFO 2026-04-06 09:57:20,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 296 |
-
INFO 2026-04-06 10:00:22,907 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 297 |
-
INFO 2026-04-06 10:03:48,884 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 298 |
-
INFO 2026-04-06 10:06:52,105 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 299 |
-
INFO 2026-04-06 10:09:59,029 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 300 |
-
INFO 2026-04-06 10:13:05,841 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 301 |
-
INFO 2026-04-06 10:16:11,935 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 302 |
-
INFO 2026-04-06 10:19:15,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 303 |
-
INFO 2026-04-06 10:22:24,669 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 304 |
-
INFO 2026-04-06 10:25:33,552 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 305 |
-
INFO 2026-04-06 10:28:42,181 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 306 |
-
INFO 2026-04-06 10:31:51,520 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 307 |
-
INFO 2026-04-06 10:35:00,447 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 308 |
-
INFO 2026-04-06 10:38:08,612 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 309 |
-
INFO 2026-04-06 10:41:15,603 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 310 |
-
INFO 2026-04-06 11:29:31,542 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=6, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 311 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 312 |
-
hyperparams=ConfigDict[typing.Any](
|
| 313 |
-
lr=5e-05,
|
| 314 |
-
weight_decay=0.0,
|
| 315 |
-
),
|
| 316 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 317 |
-
torch_optimizer='torch.optim.AdamW',
|
| 318 |
-
), scheduler=SequentialSchedulerConfig(
|
| 319 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 320 |
-
milestones=ConfigList[int](
|
| 321 |
-
500,
|
| 322 |
-
),
|
| 323 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 324 |
-
warmup=TorchSchedulerConfig(
|
| 325 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 326 |
-
hyperparams=ConfigDict[typing.Any](
|
| 327 |
-
start_factor=0.0002,
|
| 328 |
-
end_factor=1.0,
|
| 329 |
-
total_iters=500,
|
| 330 |
-
),
|
| 331 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 332 |
-
),
|
| 333 |
-
const=TorchSchedulerConfig(
|
| 334 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 335 |
-
hyperparams=ConfigDict[typing.Any](
|
| 336 |
-
factor=1.0,
|
| 337 |
-
total_iters=300000,
|
| 338 |
-
),
|
| 339 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 340 |
-
),
|
| 341 |
-
),
|
| 342 |
-
)
|
| 343 |
-
INFO 2026-04-06 11:29:31,545 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 344 |
-
INFO 2026-04-06 11:29:34,960 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 345 |
-
INFO 2026-04-06 11:29:43,764 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 346 |
-
INFO 2026-04-06 11:29:48,060 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 347 |
-
INFO 2026-04-06 11:29:51,777 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 348 |
-
INFO 2026-04-06 11:29:51,777 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=16817.3, dataset_name=bridge_paraphrase_k5
|
| 349 |
-
INFO 2026-04-06 11:29:51,818 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183295, rank=6
|
| 350 |
-
INFO 2026-04-06 11:29:52,637 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=6, status=start
|
| 351 |
-
INFO 2026-04-06 11:29:52,678 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
|
| 352 |
-
INFO 2026-04-06 11:29:52,679 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=21134.0
|
| 353 |
-
INFO 2026-04-06 11:29:52,684 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=6, status=start
|
| 354 |
-
INFO 2026-04-06 11:31:39,839 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=6, status=success
|
| 355 |
-
INFO 2026-04-06 11:32:01,842 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=6
|
| 356 |
-
INFO 2026-04-06 11:32:01,844 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 357 |
-
INFO 2026-04-06 11:32:02,000 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-7, worker_pid=3492774, parent_pid=3489446, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 358 |
-
INFO 2026-04-06 11:32:02,023 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-7, worker_pid=3492782, parent_pid=3489446, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 359 |
-
INFO 2026-04-06 11:35:29,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 360 |
-
INFO 2026-04-06 11:38:35,206 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 361 |
-
INFO 2026-04-06 11:41:41,429 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 362 |
-
INFO 2026-04-06 11:44:46,850 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 363 |
-
INFO 2026-04-06 11:47:52,316 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 364 |
-
INFO 2026-04-06 11:50:56,956 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 365 |
-
INFO 2026-04-06 11:54:03,028 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 366 |
-
INFO 2026-04-06 11:57:07,422 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 367 |
-
INFO 2026-04-06 12:00:13,667 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 368 |
-
INFO 2026-04-06 12:03:19,351 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 369 |
-
INFO 2026-04-06 12:06:23,697 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 370 |
-
INFO 2026-04-06 12:09:29,450 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 371 |
-
INFO 2026-04-06 12:12:35,575 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 372 |
-
INFO 2026-04-06 12:15:41,418 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 373 |
-
INFO 2026-04-06 12:18:46,706 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 374 |
-
INFO 2026-04-06 12:21:51,864 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 375 |
-
INFO 2026-04-06 12:24:57,746 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 376 |
-
INFO 2026-04-06 12:28:01,899 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 377 |
-
INFO 2026-04-06 12:31:07,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 378 |
-
INFO 2026-04-06 12:34:11,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 379 |
-
INFO 2026-04-06 12:37:16,741 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 380 |
-
INFO 2026-04-06 12:40:21,219 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 381 |
-
INFO 2026-04-06 12:43:25,435 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 382 |
-
INFO 2026-04-06 12:46:31,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 383 |
-
INFO 2026-04-06 12:49:37,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 384 |
-
INFO 2026-04-06 12:52:43,063 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 385 |
-
INFO 2026-04-06 12:55:47,774 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 386 |
-
INFO 2026-04-06 12:58:53,488 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 387 |
-
INFO 2026-04-06 13:01:58,055 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 388 |
-
INFO 2026-04-06 13:05:03,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 389 |
-
INFO 2026-04-06 13:08:09,159 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 390 |
-
INFO 2026-04-06 13:11:13,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 391 |
-
INFO 2026-04-06 13:14:19,369 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 392 |
-
INFO 2026-04-06 13:17:25,230 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 393 |
-
INFO 2026-04-06 13:20:30,201 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 394 |
-
INFO 2026-04-06 13:23:35,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 395 |
-
INFO 2026-04-06 13:26:41,884 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 396 |
-
INFO 2026-04-06 13:29:48,087 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 397 |
-
INFO 2026-04-06 13:32:52,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 398 |
-
INFO 2026-04-06 13:35:57,627 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 399 |
-
INFO 2026-04-06 13:39:03,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 400 |
-
INFO 2026-04-06 13:42:10,406 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 401 |
-
INFO 2026-04-06 13:45:14,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 402 |
-
INFO 2026-04-06 13:48:20,970 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 403 |
-
INFO 2026-04-06 13:51:25,882 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 404 |
-
INFO 2026-04-06 13:54:30,723 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 405 |
-
INFO 2026-04-06 13:57:36,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 406 |
-
INFO 2026-04-06 14:00:41,877 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 407 |
-
INFO 2026-04-06 14:03:49,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 408 |
-
INFO 2026-04-06 14:07:04,670 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 409 |
-
INFO 2026-04-06 14:10:39,272 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 410 |
-
INFO 2026-04-06 14:13:44,086 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 411 |
-
INFO 2026-04-06 14:16:49,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 412 |
-
INFO 2026-04-06 14:19:53,531 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 413 |
-
INFO 2026-04-06 14:22:58,790 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 414 |
-
INFO 2026-04-06 14:26:03,764 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 415 |
-
INFO 2026-04-06 14:29:07,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 416 |
-
INFO 2026-04-06 14:32:11,856 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 417 |
-
INFO 2026-04-06 14:35:16,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 418 |
-
INFO 2026-04-06 14:38:22,037 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 419 |
-
INFO 2026-04-06 14:41:25,477 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 420 |
-
INFO 2026-04-06 14:44:30,530 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 421 |
-
INFO 2026-04-06 14:47:36,056 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 422 |
-
INFO 2026-04-06 14:50:40,770 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 423 |
-
INFO 2026-04-06 14:53:45,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 424 |
-
INFO 2026-04-06 14:56:50,256 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 425 |
-
INFO 2026-04-06 14:59:56,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 426 |
-
INFO 2026-04-06 15:03:02,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 427 |
-
INFO 2026-04-07 00:41:05,408 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=6, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 428 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 429 |
-
hyperparams=ConfigDict[typing.Any](
|
| 430 |
-
lr=5e-05,
|
| 431 |
-
weight_decay=0.0,
|
| 432 |
-
),
|
| 433 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 434 |
-
torch_optimizer='torch.optim.AdamW',
|
| 435 |
-
), scheduler=SequentialSchedulerConfig(
|
| 436 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 437 |
-
milestones=ConfigList[int](
|
| 438 |
-
500,
|
| 439 |
-
),
|
| 440 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 441 |
-
warmup=TorchSchedulerConfig(
|
| 442 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 443 |
-
hyperparams=ConfigDict[typing.Any](
|
| 444 |
-
start_factor=0.0002,
|
| 445 |
-
end_factor=1.0,
|
| 446 |
-
total_iters=500,
|
| 447 |
-
),
|
| 448 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 449 |
-
),
|
| 450 |
-
const=TorchSchedulerConfig(
|
| 451 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 452 |
-
hyperparams=ConfigDict[typing.Any](
|
| 453 |
-
factor=1.0,
|
| 454 |
-
total_iters=300000,
|
| 455 |
-
),
|
| 456 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 457 |
-
),
|
| 458 |
-
),
|
| 459 |
-
)
|
| 460 |
-
INFO 2026-04-07 00:41:05,410 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 461 |
-
INFO 2026-04-07 00:41:08,936 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 462 |
-
INFO 2026-04-07 00:41:26,311 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 463 |
-
INFO 2026-04-07 00:41:36,523 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 464 |
-
INFO 2026-04-07 00:41:40,869 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 465 |
-
INFO 2026-04-07 00:41:40,869 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=31933.6, dataset_name=bridge_paraphrase_k5
|
| 466 |
-
INFO 2026-04-07 00:41:40,910 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183295, rank=6
|
| 467 |
-
INFO 2026-04-07 00:41:41,765 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=6, status=start
|
| 468 |
-
INFO 2026-04-07 00:41:41,805 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183295, num_data_pointers=164055, pct=89.50%, rank=6, status=success
|
| 469 |
-
INFO 2026-04-07 00:41:41,806 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=36396.2
|
| 470 |
-
INFO 2026-04-07 00:41:41,811 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=6, status=start
|
| 471 |
-
INFO 2026-04-07 00:43:48,407 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=6, status=success
|
| 472 |
-
INFO 2026-04-07 00:44:10,422 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=6
|
| 473 |
-
INFO 2026-04-07 00:44:10,423 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=6, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 474 |
-
INFO 2026-04-07 00:44:10,504 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=3313522, parent_pid=3310184, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 475 |
-
INFO 2026-04-07 00:44:10,548 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=6, hostname=msp3-5, worker_pid=3313561, parent_pid=3310184, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 476 |
-
INFO 2026-04-07 00:47:36,030 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 477 |
-
INFO 2026-04-07 00:50:38,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 478 |
-
INFO 2026-04-07 00:53:42,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 479 |
-
INFO 2026-04-07 00:56:45,084 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 480 |
-
INFO 2026-04-07 00:59:47,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 481 |
-
INFO 2026-04-07 01:02:50,208 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 482 |
-
INFO 2026-04-07 01:05:53,527 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 483 |
-
INFO 2026-04-07 01:08:56,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 484 |
-
INFO 2026-04-07 01:11:58,574 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 485 |
-
INFO 2026-04-07 01:15:02,200 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 486 |
-
INFO 2026-04-07 01:18:05,102 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 487 |
-
INFO 2026-04-07 01:21:08,470 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 488 |
-
INFO 2026-04-07 01:24:11,699 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 489 |
-
INFO 2026-04-07 01:27:15,143 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 490 |
-
INFO 2026-04-07 01:30:16,883 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 491 |
-
INFO 2026-04-07 01:33:19,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 492 |
-
INFO 2026-04-07 01:36:22,453 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 493 |
-
INFO 2026-04-07 01:39:25,148 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 494 |
-
INFO 2026-04-07 01:42:29,624 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 495 |
-
INFO 2026-04-07 01:45:32,635 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 496 |
-
INFO 2026-04-07 01:48:36,848 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 497 |
-
INFO 2026-04-07 01:51:39,488 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 498 |
-
INFO 2026-04-07 01:54:42,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 499 |
-
INFO 2026-04-07 01:57:46,407 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 500 |
-
INFO 2026-04-07 02:00:50,171 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 501 |
-
INFO 2026-04-07 02:03:54,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 502 |
-
INFO 2026-04-07 02:06:57,975 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 503 |
-
INFO 2026-04-07 02:10:02,506 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 504 |
-
INFO 2026-04-07 02:13:05,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 505 |
-
INFO 2026-04-07 02:16:10,139 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 506 |
-
INFO 2026-04-07 02:19:14,241 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 507 |
-
INFO 2026-04-07 02:22:17,153 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 508 |
-
INFO 2026-04-07 02:25:21,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 509 |
-
INFO 2026-04-07 02:28:24,735 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 510 |
-
INFO 2026-04-07 02:31:29,705 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 511 |
-
INFO 2026-04-07 02:34:33,433 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 512 |
-
INFO 2026-04-07 02:37:37,739 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 513 |
-
INFO 2026-04-07 02:40:41,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 514 |
-
INFO 2026-04-07 02:43:44,850 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 515 |
-
INFO 2026-04-07 02:46:48,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 516 |
-
INFO 2026-04-07 02:49:52,439 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 517 |
-
INFO 2026-04-07 02:52:56,832 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 518 |
-
INFO 2026-04-07 02:56:01,908 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 519 |
-
INFO 2026-04-07 02:59:07,330 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 520 |
-
INFO 2026-04-07 03:02:11,724 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 521 |
-
INFO 2026-04-07 03:05:16,730 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 522 |
-
INFO 2026-04-07 03:08:20,116 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 523 |
-
INFO 2026-04-07 03:11:23,981 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 524 |
-
INFO 2026-04-07 03:14:28,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 525 |
-
INFO 2026-04-07 03:17:51,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 526 |
-
INFO 2026-04-07 03:18:34,222 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=6
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:28436bacbb6ea3b7e483252210135f9d5c163b54976a0a20ff7e27b3f6b99a27
|
| 3 |
+
size 64032
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/session_rank_7.log
CHANGED
|
@@ -1,526 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
lr=5e-05,
|
| 5 |
-
weight_decay=0.0,
|
| 6 |
-
),
|
| 7 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 8 |
-
torch_optimizer='torch.optim.AdamW',
|
| 9 |
-
), scheduler=SequentialSchedulerConfig(
|
| 10 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 11 |
-
milestones=ConfigList[int](
|
| 12 |
-
500,
|
| 13 |
-
),
|
| 14 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 15 |
-
warmup=TorchSchedulerConfig(
|
| 16 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 17 |
-
hyperparams=ConfigDict[typing.Any](
|
| 18 |
-
start_factor=0.0002,
|
| 19 |
-
end_factor=1.0,
|
| 20 |
-
total_iters=500,
|
| 21 |
-
),
|
| 22 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 23 |
-
),
|
| 24 |
-
const=TorchSchedulerConfig(
|
| 25 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 26 |
-
hyperparams=ConfigDict[typing.Any](
|
| 27 |
-
factor=1.0,
|
| 28 |
-
total_iters=300000,
|
| 29 |
-
),
|
| 30 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 31 |
-
),
|
| 32 |
-
),
|
| 33 |
-
)
|
| 34 |
-
INFO 2026-04-05 07:47:39,915 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 35 |
-
INFO 2026-04-05 07:47:42,038 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge
|
| 36 |
-
INFO 2026-04-05 07:48:03,856 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 37 |
-
INFO 2026-04-05 07:48:08,037 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 38 |
-
INFO 2026-04-05 07:48:08,037 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=25998.7, dataset_name=bridge
|
| 39 |
-
INFO 2026-04-05 07:48:08,077 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge, size_before=1466050, size_after=183242, rank=7
|
| 40 |
-
INFO 2026-04-05 07:48:08,930 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge rank=7, status=start
|
| 41 |
-
INFO 2026-04-05 07:48:08,969 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
|
| 42 |
-
INFO 2026-04-05 07:48:08,970 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=29055.3
|
| 43 |
-
INFO 2026-04-05 07:48:34,458 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=7
|
| 44 |
-
INFO 2026-04-05 07:48:34,460 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 45 |
-
INFO 2026-04-05 07:48:34,515 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-2, worker_pid=3875748, parent_pid=3872896, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 46 |
-
INFO 2026-04-05 07:48:34,543 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-2, worker_pid=3875758, parent_pid=3872896, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 47 |
-
INFO 2026-04-05 07:51:52,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 48 |
-
INFO 2026-04-05 07:54:44,997 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 49 |
-
INFO 2026-04-05 07:57:35,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 50 |
-
INFO 2026-04-05 08:00:26,862 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 51 |
-
INFO 2026-04-05 08:03:18,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 52 |
-
INFO 2026-04-05 08:06:09,619 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 53 |
-
INFO 2026-04-05 08:09:01,077 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 54 |
-
INFO 2026-04-05 08:11:53,412 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 55 |
-
INFO 2026-04-05 08:14:44,578 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 56 |
-
INFO 2026-04-05 08:17:37,375 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 57 |
-
INFO 2026-04-05 08:20:30,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 58 |
-
INFO 2026-04-05 08:23:23,222 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 59 |
-
INFO 2026-04-05 08:26:15,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 60 |
-
INFO 2026-04-05 08:29:07,502 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 61 |
-
INFO 2026-04-05 08:31:59,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 62 |
-
INFO 2026-04-05 08:34:50,984 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 63 |
-
INFO 2026-04-05 08:37:43,456 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 64 |
-
INFO 2026-04-05 08:40:35,332 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 65 |
-
INFO 2026-04-05 08:43:28,467 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 66 |
-
INFO 2026-04-05 08:46:21,132 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 67 |
-
INFO 2026-04-05 08:49:13,210 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 68 |
-
INFO 2026-04-05 08:52:04,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 69 |
-
INFO 2026-04-05 08:54:56,866 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 70 |
-
INFO 2026-04-05 08:57:48,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 71 |
-
INFO 2026-04-05 09:00:40,238 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 72 |
-
INFO 2026-04-05 09:03:32,325 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 73 |
-
INFO 2026-04-05 09:06:23,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 74 |
-
INFO 2026-04-05 09:09:15,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 75 |
-
INFO 2026-04-05 09:12:08,117 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 76 |
-
INFO 2026-04-05 09:14:59,227 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 77 |
-
INFO 2026-04-05 09:17:51,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 78 |
-
INFO 2026-04-05 09:20:42,298 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 79 |
-
INFO 2026-04-05 09:23:34,083 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 80 |
-
INFO 2026-04-05 09:26:26,064 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 81 |
-
INFO 2026-04-05 09:29:18,322 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 82 |
-
INFO 2026-04-05 09:32:09,487 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 83 |
-
INFO 2026-04-05 09:35:00,441 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 84 |
-
INFO 2026-04-05 09:37:52,631 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 85 |
-
INFO 2026-04-05 09:40:43,428 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 86 |
-
INFO 2026-04-05 09:43:34,586 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 87 |
-
INFO 2026-04-05 09:46:26,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 88 |
-
INFO 2026-04-05 09:49:17,267 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 89 |
-
INFO 2026-04-05 09:52:06,854 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 90 |
-
INFO 2026-04-05 09:54:57,889 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 91 |
-
INFO 2026-04-05 09:57:48,894 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 92 |
-
INFO 2026-04-05 10:00:41,039 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 93 |
-
INFO 2026-04-05 10:03:33,426 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 94 |
-
INFO 2026-04-05 10:06:24,800 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 95 |
-
INFO 2026-04-05 10:09:17,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 96 |
-
INFO 2026-04-05 10:12:18,804 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 97 |
-
INFO 2026-04-05 10:15:33,483 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 98 |
-
INFO 2026-04-05 10:18:24,744 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 99 |
-
INFO 2026-04-05 10:21:17,302 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 100 |
-
INFO 2026-04-05 10:24:08,917 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 101 |
-
INFO 2026-04-05 10:27:00,005 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 102 |
-
INFO 2026-04-05 10:29:51,959 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 103 |
-
INFO 2026-04-05 10:32:42,383 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 104 |
-
INFO 2026-04-05 10:35:34,045 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 105 |
-
INFO 2026-04-05 10:38:25,583 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 106 |
-
INFO 2026-04-05 10:41:18,350 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 107 |
-
INFO 2026-04-05 10:44:08,990 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 108 |
-
INFO 2026-04-05 10:47:00,701 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 109 |
-
INFO 2026-04-05 10:49:51,287 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 110 |
-
INFO 2026-04-05 10:52:43,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 111 |
-
INFO 2026-04-05 10:55:35,867 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 112 |
-
INFO 2026-04-05 10:58:26,949 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 113 |
-
INFO 2026-04-05 11:01:18,366 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 114 |
-
INFO 2026-04-05 11:04:10,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 115 |
-
INFO 2026-04-05 11:07:02,461 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 116 |
-
INFO 2026-04-05 11:09:53,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 117 |
-
INFO 2026-04-05 11:12:45,451 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 118 |
-
INFO 2026-04-05 11:15:36,492 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 119 |
-
INFO 2026-04-05 11:18:28,400 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 120 |
-
INFO 2026-04-05 11:21:20,006 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 121 |
-
INFO 2026-04-05 11:24:11,858 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 122 |
-
INFO 2026-04-05 11:27:02,198 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 123 |
-
INFO 2026-04-05 11:29:53,736 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 124 |
-
INFO 2026-04-05 11:32:44,011 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 125 |
-
INFO 2026-04-05 11:35:35,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 126 |
-
INFO 2026-04-05 11:38:25,486 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 127 |
-
INFO 2026-04-05 11:41:15,315 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 128 |
-
INFO 2026-04-05 11:44:05,134 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 129 |
-
INFO 2026-04-05 11:46:56,617 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 130 |
-
INFO 2026-04-05 11:49:47,469 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 131 |
-
INFO 2026-04-05 11:52:38,563 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 132 |
-
INFO 2026-04-05 11:55:30,289 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 133 |
-
INFO 2026-04-05 11:58:21,027 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 134 |
-
INFO 2026-04-05 12:01:13,424 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 135 |
-
INFO 2026-04-05 12:04:04,015 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 136 |
-
INFO 2026-04-05 12:06:55,196 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 137 |
-
INFO 2026-04-05 12:09:46,718 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 138 |
-
INFO 2026-04-05 12:12:37,434 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 139 |
-
INFO 2026-04-05 12:15:28,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 140 |
-
INFO 2026-04-05 12:18:19,903 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 141 |
-
INFO 2026-04-05 12:21:11,565 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 142 |
-
INFO 2026-04-05 12:24:02,399 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 143 |
-
INFO 2026-04-05 12:26:54,834 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 144 |
-
INFO 2026-04-05 12:29:45,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 145 |
-
INFO 2026-04-05 12:32:36,286 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 146 |
-
INFO 2026-04-05 12:35:36,951 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 147 |
-
INFO 2026-04-05 12:38:50,026 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 148 |
-
INFO 2026-04-05 12:41:41,343 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 149 |
-
INFO 2026-04-05 12:44:32,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 150 |
-
INFO 2026-04-05 12:47:23,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 151 |
-
INFO 2026-04-05 12:50:13,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 152 |
-
INFO 2026-04-05 12:53:06,274 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 153 |
-
INFO 2026-04-05 12:55:56,980 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 154 |
-
INFO 2026-04-05 12:58:48,716 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 155 |
-
INFO 2026-04-05 13:01:39,887 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 156 |
-
INFO 2026-04-05 13:04:31,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 157 |
-
INFO 2026-04-05 13:07:22,319 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 158 |
-
INFO 2026-04-05 13:10:15,398 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 159 |
-
INFO 2026-04-05 13:13:06,519 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 160 |
-
INFO 2026-04-05 13:15:57,164 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 161 |
-
INFO 2026-04-05 13:18:49,647 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 162 |
-
INFO 2026-04-05 13:21:40,162 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 163 |
-
INFO 2026-04-05 13:24:31,281 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 164 |
-
INFO 2026-04-05 13:27:22,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 165 |
-
INFO 2026-04-05 13:30:13,985 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 166 |
-
INFO 2026-04-05 13:33:06,043 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 167 |
-
INFO 2026-04-05 13:35:57,606 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 168 |
-
INFO 2026-04-05 13:38:47,618 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 169 |
-
INFO 2026-04-05 13:41:40,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 170 |
-
INFO 2026-04-05 13:44:31,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 171 |
-
INFO 2026-04-05 13:47:23,254 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 172 |
-
INFO 2026-04-05 13:50:14,660 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 173 |
-
INFO 2026-04-05 13:53:06,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 174 |
-
INFO 2026-04-05 13:55:58,107 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 175 |
-
INFO 2026-04-05 13:58:50,688 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 176 |
-
INFO 2026-04-05 14:01:41,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 177 |
-
INFO 2026-04-05 14:04:32,278 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 178 |
-
INFO 2026-04-05 14:07:24,300 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 179 |
-
INFO 2026-04-05 14:10:15,785 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 180 |
-
INFO 2026-04-05 14:13:07,365 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 181 |
-
INFO 2026-04-05 14:15:58,762 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 182 |
-
INFO 2026-04-05 14:18:49,976 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 183 |
-
INFO 2026-04-05 14:21:40,911 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 184 |
-
INFO 2026-04-05 14:24:31,803 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 185 |
-
INFO 2026-04-05 14:27:22,607 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 186 |
-
INFO 2026-04-05 14:30:14,379 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 187 |
-
INFO 2026-04-05 14:33:06,049 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 188 |
-
INFO 2026-04-05 14:35:56,094 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 189 |
-
INFO 2026-04-05 14:38:48,561 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 190 |
-
INFO 2026-04-05 14:41:40,352 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 191 |
-
INFO 2026-04-05 14:44:32,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 192 |
-
INFO 2026-04-05 14:47:25,215 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 193 |
-
INFO 2026-04-05 14:50:15,915 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 194 |
-
INFO 2026-04-05 14:53:07,347 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 195 |
-
INFO 2026-04-05 14:56:00,277 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 196 |
-
INFO 2026-04-05 14:59:02,313 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 197 |
-
INFO 2026-04-05 14:59:35,376 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=7
|
| 198 |
-
INFO 2026-04-06 07:33:53,243 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=7, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 199 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 200 |
-
hyperparams=ConfigDict[typing.Any](
|
| 201 |
-
lr=5e-05,
|
| 202 |
-
weight_decay=0.0,
|
| 203 |
-
),
|
| 204 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 205 |
-
torch_optimizer='torch.optim.AdamW',
|
| 206 |
-
), scheduler=SequentialSchedulerConfig(
|
| 207 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 208 |
-
milestones=ConfigList[int](
|
| 209 |
-
500,
|
| 210 |
-
),
|
| 211 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 212 |
-
warmup=TorchSchedulerConfig(
|
| 213 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 214 |
-
hyperparams=ConfigDict[typing.Any](
|
| 215 |
-
start_factor=0.0002,
|
| 216 |
-
end_factor=1.0,
|
| 217 |
-
total_iters=500,
|
| 218 |
-
),
|
| 219 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 220 |
-
),
|
| 221 |
-
const=TorchSchedulerConfig(
|
| 222 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 223 |
-
hyperparams=ConfigDict[typing.Any](
|
| 224 |
-
factor=1.0,
|
| 225 |
-
total_iters=300000,
|
| 226 |
-
),
|
| 227 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 228 |
-
),
|
| 229 |
-
),
|
| 230 |
-
)
|
| 231 |
-
INFO 2026-04-06 07:33:53,246 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 232 |
-
INFO 2026-04-06 07:33:56,074 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 233 |
-
INFO 2026-04-06 07:34:10,300 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 234 |
-
INFO 2026-04-06 07:34:17,978 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 235 |
-
INFO 2026-04-06 07:34:22,270 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 236 |
-
INFO 2026-04-06 07:34:22,270 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=26196.8, dataset_name=bridge_paraphrase_k5
|
| 237 |
-
INFO 2026-04-06 07:34:22,313 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183242, rank=7
|
| 238 |
-
INFO 2026-04-06 07:34:23,188 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=7, status=start
|
| 239 |
-
INFO 2026-04-06 07:34:23,228 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
|
| 240 |
-
INFO 2026-04-06 07:34:23,230 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=29983.399999999998
|
| 241 |
-
INFO 2026-04-06 07:34:23,236 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=7, status=start
|
| 242 |
-
INFO 2026-04-06 07:36:05,844 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=7, status=success
|
| 243 |
-
INFO 2026-04-06 07:36:27,876 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=7
|
| 244 |
-
INFO 2026-04-06 07:36:27,879 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 245 |
-
INFO 2026-04-06 07:36:27,946 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=2465171, parent_pid=2461892, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 246 |
-
INFO 2026-04-06 07:36:27,979 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=2465202, parent_pid=2461892, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 247 |
-
INFO 2026-04-06 07:39:43,068 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 248 |
-
INFO 2026-04-06 07:42:35,329 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 249 |
-
INFO 2026-04-06 07:45:26,709 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 250 |
-
INFO 2026-04-06 07:48:18,405 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 251 |
-
INFO 2026-04-06 07:51:08,588 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 252 |
-
INFO 2026-04-06 07:54:00,463 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 253 |
-
INFO 2026-04-06 07:56:51,070 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 254 |
-
INFO 2026-04-06 07:59:43,648 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 255 |
-
INFO 2026-04-06 08:02:34,655 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 256 |
-
INFO 2026-04-06 08:05:26,546 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 257 |
-
INFO 2026-04-06 08:08:18,653 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 258 |
-
INFO 2026-04-06 08:11:11,326 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 259 |
-
INFO 2026-04-06 08:14:03,148 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 260 |
-
INFO 2026-04-06 08:16:57,119 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 261 |
-
INFO 2026-04-06 08:19:48,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 262 |
-
INFO 2026-04-06 08:22:39,880 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 263 |
-
INFO 2026-04-06 08:25:31,510 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 264 |
-
INFO 2026-04-06 08:28:25,199 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 265 |
-
INFO 2026-04-06 08:31:16,666 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 266 |
-
INFO 2026-04-06 08:34:09,668 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 267 |
-
INFO 2026-04-06 08:37:01,040 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 268 |
-
INFO 2026-04-06 08:39:53,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 269 |
-
INFO 2026-04-06 08:42:45,381 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 270 |
-
INFO 2026-04-06 08:45:38,330 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 271 |
-
INFO 2026-04-06 08:48:30,988 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 272 |
-
INFO 2026-04-06 08:51:23,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 273 |
-
INFO 2026-04-06 08:54:13,610 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 274 |
-
INFO 2026-04-06 08:57:05,895 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 275 |
-
INFO 2026-04-06 08:59:57,948 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 276 |
-
INFO 2026-04-06 09:02:49,534 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 277 |
-
INFO 2026-04-06 09:05:41,813 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 278 |
-
INFO 2026-04-06 09:08:33,644 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 279 |
-
INFO 2026-04-06 09:11:25,220 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 280 |
-
INFO 2026-04-06 09:14:17,223 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 281 |
-
INFO 2026-04-06 09:17:08,793 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 282 |
-
INFO 2026-04-06 09:20:01,791 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 283 |
-
INFO 2026-04-06 09:22:53,269 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 284 |
-
INFO 2026-04-06 09:25:46,358 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 285 |
-
INFO 2026-04-06 09:28:40,722 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 286 |
-
INFO 2026-04-06 09:31:33,388 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 287 |
-
INFO 2026-04-06 09:34:24,732 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 288 |
-
INFO 2026-04-06 09:37:17,501 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 289 |
-
INFO 2026-04-06 09:40:08,460 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 290 |
-
INFO 2026-04-06 09:43:00,698 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 291 |
-
INFO 2026-04-06 09:45:52,659 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 292 |
-
INFO 2026-04-06 09:48:44,228 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 293 |
-
INFO 2026-04-06 09:51:36,496 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 294 |
-
INFO 2026-04-06 09:54:29,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 295 |
-
INFO 2026-04-06 09:57:20,633 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 296 |
-
INFO 2026-04-06 10:00:22,925 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 297 |
-
INFO 2026-04-06 10:03:48,883 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 298 |
-
INFO 2026-04-06 10:06:52,079 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 299 |
-
INFO 2026-04-06 10:09:59,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 300 |
-
INFO 2026-04-06 10:13:05,838 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 301 |
-
INFO 2026-04-06 10:16:11,930 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 302 |
-
INFO 2026-04-06 10:19:15,974 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 303 |
-
INFO 2026-04-06 10:22:24,663 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 304 |
-
INFO 2026-04-06 10:25:33,547 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 305 |
-
INFO 2026-04-06 10:28:42,171 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 306 |
-
INFO 2026-04-06 10:31:51,514 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 307 |
-
INFO 2026-04-06 10:35:00,444 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 308 |
-
INFO 2026-04-06 10:38:08,605 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 309 |
-
INFO 2026-04-06 10:41:15,596 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 310 |
-
INFO 2026-04-06 11:29:31,530 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=7, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 311 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 312 |
-
hyperparams=ConfigDict[typing.Any](
|
| 313 |
-
lr=5e-05,
|
| 314 |
-
weight_decay=0.0,
|
| 315 |
-
),
|
| 316 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 317 |
-
torch_optimizer='torch.optim.AdamW',
|
| 318 |
-
), scheduler=SequentialSchedulerConfig(
|
| 319 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 320 |
-
milestones=ConfigList[int](
|
| 321 |
-
500,
|
| 322 |
-
),
|
| 323 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 324 |
-
warmup=TorchSchedulerConfig(
|
| 325 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 326 |
-
hyperparams=ConfigDict[typing.Any](
|
| 327 |
-
start_factor=0.0002,
|
| 328 |
-
end_factor=1.0,
|
| 329 |
-
total_iters=500,
|
| 330 |
-
),
|
| 331 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 332 |
-
),
|
| 333 |
-
const=TorchSchedulerConfig(
|
| 334 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 335 |
-
hyperparams=ConfigDict[typing.Any](
|
| 336 |
-
factor=1.0,
|
| 337 |
-
total_iters=300000,
|
| 338 |
-
),
|
| 339 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 340 |
-
),
|
| 341 |
-
),
|
| 342 |
-
)
|
| 343 |
-
INFO 2026-04-06 11:29:31,533 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 344 |
-
INFO 2026-04-06 11:29:34,898 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 345 |
-
INFO 2026-04-06 11:29:43,471 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 346 |
-
INFO 2026-04-06 11:29:47,725 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 347 |
-
INFO 2026-04-06 11:29:51,391 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 348 |
-
INFO 2026-04-06 11:29:51,391 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=16493.5, dataset_name=bridge_paraphrase_k5
|
| 349 |
-
INFO 2026-04-06 11:29:51,433 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183242, rank=7
|
| 350 |
-
INFO 2026-04-06 11:29:52,254 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=7, status=start
|
| 351 |
-
INFO 2026-04-06 11:29:52,293 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
|
| 352 |
-
INFO 2026-04-06 11:29:52,295 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=20761.7
|
| 353 |
-
INFO 2026-04-06 11:29:52,300 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=7, status=start
|
| 354 |
-
INFO 2026-04-06 11:31:39,840 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=7, status=success
|
| 355 |
-
INFO 2026-04-06 11:32:01,842 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=7
|
| 356 |
-
INFO 2026-04-06 11:32:01,844 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 357 |
-
INFO 2026-04-06 11:32:02,002 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-7, worker_pid=3492776, parent_pid=3489447, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 358 |
-
INFO 2026-04-06 11:32:02,028 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-7, worker_pid=3492784, parent_pid=3489447, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 359 |
-
INFO 2026-04-06 11:35:29,327 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 360 |
-
INFO 2026-04-06 11:38:35,202 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 361 |
-
INFO 2026-04-06 11:41:41,424 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 362 |
-
INFO 2026-04-06 11:44:46,845 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 363 |
-
INFO 2026-04-06 11:47:52,312 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 364 |
-
INFO 2026-04-06 11:50:56,955 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 365 |
-
INFO 2026-04-06 11:54:03,038 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 366 |
-
INFO 2026-04-06 11:57:07,421 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 367 |
-
INFO 2026-04-06 12:00:13,663 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 368 |
-
INFO 2026-04-06 12:03:19,349 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 369 |
-
INFO 2026-04-06 12:06:23,693 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 370 |
-
INFO 2026-04-06 12:09:29,444 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 371 |
-
INFO 2026-04-06 12:12:35,568 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 372 |
-
INFO 2026-04-06 12:15:41,415 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 373 |
-
INFO 2026-04-06 12:18:46,702 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 374 |
-
INFO 2026-04-06 12:21:51,861 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 375 |
-
INFO 2026-04-06 12:24:57,746 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 376 |
-
INFO 2026-04-06 12:28:01,896 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 377 |
-
INFO 2026-04-06 12:31:07,226 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 378 |
-
INFO 2026-04-06 12:34:11,865 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 379 |
-
INFO 2026-04-06 12:37:16,746 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 380 |
-
INFO 2026-04-06 12:40:21,216 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 381 |
-
INFO 2026-04-06 12:43:25,432 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 382 |
-
INFO 2026-04-06 12:46:31,790 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 383 |
-
INFO 2026-04-06 12:49:37,506 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 384 |
-
INFO 2026-04-06 12:52:43,059 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 385 |
-
INFO 2026-04-06 12:55:47,770 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 386 |
-
INFO 2026-04-06 12:58:53,484 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 387 |
-
INFO 2026-04-06 13:01:58,050 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 388 |
-
INFO 2026-04-06 13:05:03,711 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 389 |
-
INFO 2026-04-06 13:08:09,154 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 390 |
-
INFO 2026-04-06 13:11:13,196 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 391 |
-
INFO 2026-04-06 13:14:19,368 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 392 |
-
INFO 2026-04-06 13:17:25,217 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 393 |
-
INFO 2026-04-06 13:20:30,196 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 394 |
-
INFO 2026-04-06 13:23:35,853 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 395 |
-
INFO 2026-04-06 13:26:41,878 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 396 |
-
INFO 2026-04-06 13:29:48,080 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 397 |
-
INFO 2026-04-06 13:32:52,614 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 398 |
-
INFO 2026-04-06 13:35:57,628 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 399 |
-
INFO 2026-04-06 13:39:03,078 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 400 |
-
INFO 2026-04-06 13:42:10,402 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 401 |
-
INFO 2026-04-06 13:45:14,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 402 |
-
INFO 2026-04-06 13:48:20,996 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 403 |
-
INFO 2026-04-06 13:51:25,869 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 404 |
-
INFO 2026-04-06 13:54:30,720 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 405 |
-
INFO 2026-04-06 13:57:36,002 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 406 |
-
INFO 2026-04-06 14:00:41,873 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 407 |
-
INFO 2026-04-06 14:03:49,079 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 408 |
-
INFO 2026-04-06 14:07:04,763 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 409 |
-
INFO 2026-04-06 14:10:39,266 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 410 |
-
INFO 2026-04-06 14:13:44,078 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 411 |
-
INFO 2026-04-06 14:16:49,282 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 412 |
-
INFO 2026-04-06 14:19:53,530 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 413 |
-
INFO 2026-04-06 14:22:58,776 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 414 |
-
INFO 2026-04-06 14:26:03,758 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 415 |
-
INFO 2026-04-06 14:29:07,808 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 416 |
-
INFO 2026-04-06 14:32:11,851 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 417 |
-
INFO 2026-04-06 14:35:16,308 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 418 |
-
INFO 2026-04-06 14:38:22,033 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 419 |
-
INFO 2026-04-06 14:41:25,485 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 420 |
-
INFO 2026-04-06 14:44:30,526 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 421 |
-
INFO 2026-04-06 14:47:36,041 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 422 |
-
INFO 2026-04-06 14:50:40,763 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 423 |
-
INFO 2026-04-06 14:53:45,032 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 424 |
-
INFO 2026-04-06 14:56:50,250 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 425 |
-
INFO 2026-04-06 14:59:56,684 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 426 |
-
INFO 2026-04-06 15:03:02,320 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 427 |
-
INFO 2026-04-07 00:41:05,444 [1;1mbarrel.train.components.optimizers.optimizer_module[1;0m init_optimizer rank=7, num_tensors=766, groups={'default': 766}, optimizer=TorchOptimizerConfig(
|
| 428 |
-
autoclass='barrel.train.components.optimizers.optimizers.torch_optimizer.TorchOptimizer',
|
| 429 |
-
hyperparams=ConfigDict[typing.Any](
|
| 430 |
-
lr=5e-05,
|
| 431 |
-
weight_decay=0.0,
|
| 432 |
-
),
|
| 433 |
-
param_groups=ConfigDict[typing.Dict[str, typing.Any]](),
|
| 434 |
-
torch_optimizer='torch.optim.AdamW',
|
| 435 |
-
), scheduler=SequentialSchedulerConfig(
|
| 436 |
-
autoclass='barrel.train.components.optimizers.schedulers.sequential_scheduler.SequentialScheduler',
|
| 437 |
-
milestones=ConfigList[int](
|
| 438 |
-
500,
|
| 439 |
-
),
|
| 440 |
-
schedulers=ConfigDict[AutoConfig[SchedulerConfig]](
|
| 441 |
-
warmup=TorchSchedulerConfig(
|
| 442 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 443 |
-
hyperparams=ConfigDict[typing.Any](
|
| 444 |
-
start_factor=0.0002,
|
| 445 |
-
end_factor=1.0,
|
| 446 |
-
total_iters=500,
|
| 447 |
-
),
|
| 448 |
-
torch_scheduler='torch.optim.lr_scheduler.LinearLR',
|
| 449 |
-
),
|
| 450 |
-
const=TorchSchedulerConfig(
|
| 451 |
-
autoclass='barrel.train.components.optimizers.schedulers.torch_scheduler.TorchScheduler',
|
| 452 |
-
hyperparams=ConfigDict[typing.Any](
|
| 453 |
-
factor=1.0,
|
| 454 |
-
total_iters=300000,
|
| 455 |
-
),
|
| 456 |
-
torch_scheduler='torch.optim.lr_scheduler.ConstantLR',
|
| 457 |
-
),
|
| 458 |
-
),
|
| 459 |
-
)
|
| 460 |
-
INFO 2026-04-07 00:41:05,448 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=start
|
| 461 |
-
INFO 2026-04-07 00:41:08,589 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=start, dataset_name=bridge_paraphrase_k5
|
| 462 |
-
INFO 2026-04-07 00:41:18,091 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.transforms[1;0m TREAD columns detected in dataset, will replace language instructions dataset_name=bridge_paraphrase_k5
|
| 463 |
-
INFO 2026-04-07 00:41:23,206 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Filtered episodes with no movement dataset_name=bridge_paraphrase_k5, size_before=1524249, size_after=1517010, keep_pct=99.53%
|
| 464 |
-
INFO 2026-04-07 00:41:27,430 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.utils.filter[1;0m Filtered frames with no movement dataset_name=bridge_paraphrase_k5, size_before=1517010, size_after=1466050, keep_pct=96.64%
|
| 465 |
-
INFO 2026-04-07 00:41:27,430 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m apply_transforms status=success, duration_ms=18840.6, dataset_name=bridge_paraphrase_k5
|
| 466 |
-
INFO 2026-04-07 00:41:27,471 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_data_source[1;0m Sharded dataset dataset_name=bridge_paraphrase_k5, size_before=1466050, size_after=183242, rank=7
|
| 467 |
-
INFO 2026-04-07 00:41:28,495 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Computing data pointers for bridge_paraphrase_k5 rank=7, status=start
|
| 468 |
-
INFO 2026-04-07 00:41:28,534 [1;1mbarrel.train.pipes.vlams.data.robotics.hf.robotics_dataset[1;0m Data pointers for bridge_paraphrase_k5 dataframe_size=183242, num_data_pointers=160418, pct=87.54%, rank=7, status=success
|
| 469 |
-
INFO 2026-04-07 00:41:28,536 [1;1mbarrel.train.pipes.vlams.pipe.oxe_data_module[1;0m load_train_dataset status=success, duration_ms=23088.3
|
| 470 |
-
INFO 2026-04-07 00:41:28,541 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint checkpoint_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1/checkpoints/checkpoint-latest.pt, rank=7, status=start
|
| 471 |
-
INFO 2026-04-07 00:43:48,421 [1;1mbarrel.train.core.trainer.trainer[1;0m Restore session checkpoint rank=7, status=success
|
| 472 |
-
INFO 2026-04-07 00:44:10,422 [1;1mbarrel.train.core.trainer.trainer[1;0m session_init status=success, rank=7
|
| 473 |
-
INFO 2026-04-07 00:44:10,423 [1;1mbarrel.train.core.trainer.trainer[1;0m start_training rank=7, session_id=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1
|
| 474 |
-
INFO 2026-04-07 00:44:10,488 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=3313511, parent_pid=3310185, worker_id=0/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 475 |
-
INFO 2026-04-07 00:44:10,525 [1;1mbarrel.train.components.data.torch_dataloader[1;0m Starting dataloader worker rank=7, hostname=msp3-5, worker_pid=3313538, parent_pid=3310185, worker_id=1/2, cpu_affinity=0-31,48-79,96-127,144-175
|
| 476 |
-
INFO 2026-04-07 00:47:36,031 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 477 |
-
INFO 2026-04-07 00:50:38,874 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 478 |
-
INFO 2026-04-07 00:53:42,316 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 479 |
-
INFO 2026-04-07 00:56:45,089 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 480 |
-
INFO 2026-04-07 00:59:47,992 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 481 |
-
INFO 2026-04-07 01:02:50,213 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 482 |
-
INFO 2026-04-07 01:05:53,531 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 483 |
-
INFO 2026-04-07 01:08:56,073 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 484 |
-
INFO 2026-04-07 01:11:58,579 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 485 |
-
INFO 2026-04-07 01:15:02,216 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 486 |
-
INFO 2026-04-07 01:18:05,106 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 487 |
-
INFO 2026-04-07 01:21:08,476 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 488 |
-
INFO 2026-04-07 01:24:11,705 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 489 |
-
INFO 2026-04-07 01:27:15,146 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 490 |
-
INFO 2026-04-07 01:30:16,888 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 491 |
-
INFO 2026-04-07 01:33:19,438 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 492 |
-
INFO 2026-04-07 01:36:22,459 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 493 |
-
INFO 2026-04-07 01:39:25,147 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 494 |
-
INFO 2026-04-07 01:42:29,626 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 495 |
-
INFO 2026-04-07 01:45:32,639 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 496 |
-
INFO 2026-04-07 01:48:36,841 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 497 |
-
INFO 2026-04-07 01:51:39,495 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 498 |
-
INFO 2026-04-07 01:54:42,918 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 499 |
-
INFO 2026-04-07 01:57:46,413 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 500 |
-
INFO 2026-04-07 02:00:50,175 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 501 |
-
INFO 2026-04-07 02:03:54,285 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 502 |
-
INFO 2026-04-07 02:06:57,978 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 503 |
-
INFO 2026-04-07 02:10:02,509 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 504 |
-
INFO 2026-04-07 02:13:05,977 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 505 |
-
INFO 2026-04-07 02:16:10,143 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 506 |
-
INFO 2026-04-07 02:19:14,244 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 507 |
-
INFO 2026-04-07 02:22:17,143 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 508 |
-
INFO 2026-04-07 02:25:21,418 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 509 |
-
INFO 2026-04-07 02:28:24,740 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 510 |
-
INFO 2026-04-07 02:31:29,710 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 511 |
-
INFO 2026-04-07 02:34:33,439 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 512 |
-
INFO 2026-04-07 02:37:37,745 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 513 |
-
INFO 2026-04-07 02:40:41,393 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 514 |
-
INFO 2026-04-07 02:43:44,857 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 515 |
-
INFO 2026-04-07 02:46:48,390 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 516 |
-
INFO 2026-04-07 02:49:52,448 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 517 |
-
INFO 2026-04-07 02:52:56,836 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 518 |
-
INFO 2026-04-07 02:56:01,914 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 519 |
-
INFO 2026-04-07 02:59:07,333 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 520 |
-
INFO 2026-04-07 03:02:11,728 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 521 |
-
INFO 2026-04-07 03:05:16,730 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 522 |
-
INFO 2026-04-07 03:08:20,104 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 523 |
-
INFO 2026-04-07 03:11:23,983 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 524 |
-
INFO 2026-04-07 03:14:28,690 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 525 |
-
INFO 2026-04-07 03:17:51,497 [1;1mbarrel.train.pipes.vlams.pipe.robotics_vis_callback[1;0m visualize_train_data status=model generate
|
| 526 |
-
INFO 2026-04-07 03:18:34,222 [1;1mbarrel.train.core.trainer.trainer[1;0m training_finished session_path=/home/petko_petkov/experiments/vlams/control/sess_2026_04_05_07_47_08_msp3-2_petko_petkov_bridge_curriculum_phase1, rank=7
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d91786f01dd1b0812225c748c18bb066833412534bd95d24015d446350ba4ae4
|
| 3 |
+
size 64010
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|