| name: vl-jepa-pretrain-video | |
| run_root: runs | |
| model: | |
| stride: 3 | |
| num_frames: 16 | |
| vision_config: | |
| name: vjepa2_1_vit_large_384 | |
| load_from: facebookresearch/vjepa2 | |
| source: github | |
| video_mode: true | |
| use_sdpa: true | |
| use_rope: true | |
| use_activation_checkpointing: true | |
| dtype: bfloat16 | |
| patch_size: 16 | |
| tubelet_size: 2 | |
| crop_size: 384 | |
| unload: true | |
| freeze: true | |
| predictor: | |
| language: | |
| model: meta-llama/Llama-3.2-1B | |
| num_layers: 4 | |
| max_query_length: 512 | |
| unload: false | |
| freeze: false | |
| compressor: | |
| type: qformer | |
| num_query_tokens: 512 | |
| num_layers: 4 | |
| num_heads: 8 | |
| mlp_ratio: 4.0 | |
| dropout: 0.2 | |
| cross_attention_frequency: 1 | |
| lr_multiplier: 2.0 | |
| target: | |
| cosmos_dir: null | |
| reason1_ckpt: nvidia/Cosmos-Reason1-7B | |
| crossattn_proj_path: data/wts_cache/crossattn_proj.pt | |
| embedding_concat_strategy: full_concat | |
| n_layers_per_group: 5 | |
| lr_multiplier: 0.05 | |
| unload: true | |
| freeze: true | |
| decoder: | |
| model: null | |
| unload: true | |
| freeze: false | |
| runtime: | |
| trust_remote_code: true | |
| hf_token: false | |
| torch_dtype: auto | |
| tiny: false | |
| embedding_dim: 1024 | |
| temperature: 0.1 | |
| data: | |
| train_file: data/vl_jepa/train.jsonl | |
| eval_file: data/vl_jepa/val.jsonl | |
| train_fraction: 1.0 | |
| query_override: null | |
| target_type: short | |
| max_target_length: 512 | |
| training: | |
| runtime: | |
| seed: 42 | |
| deterministic: true | |
| num_workers: 24 | |
| checkpoint: | |
| init_checkpoint: null | |
| init_checkpoint_run_name: null | |
| require_init_checkpoint: false | |
| epochs: 10 | |
| patience: 5 | |
| min_delta: 0.0 | |
| early_stop: true | |
| schedule: | |
| num_epochs: 50 | |
| batching: | |
| batch_size: 60 | |
| contrastive_accum_batches: 1 | |
| grad_accum: 1 | |
| optimization: | |
| type: adamw | |
| learning_rate: 4.0e-05 | |
| weight_decay: 0.04 | |
| max_grad_norm: 10.0 | |
| scheduler: | |
| type: cosine | |
| min_learning_rate: 1.0e-06 | |
| warmup_ratio: 0.05 | |
| loss: | |
| label_smoothing: 0.05 | |
| enabled: set_match:mse=1.0 | |
| center: false | |
| target_mean_path: data/wts_cache/target_mean.pt | |
| ema: | |
| enabled: false | |
| decay: | |
| - 0.99 | |
| - 0.9999 | |
| logging: | |
| tensorboard: true | |
| log_steps: 10 | |
| progress_type: tqdm | |
| save_csv: true | |
| save_batch_csv: true | |
| save_epoch_csv: true | |
| log_batch_scalars: true | |
| evaluation: | |
| epochs: 1 | |
| max_batches: null | |
| eval: | |
| pad_zero_threshold: 10.0 # seq-infer zeroes tokens with L2 norm below this (pad/null tail) |