name: vl-jepa-pretrain-video
run_root: runs
model:
  stride: 3
  num_frames: 16
  vision_config:
    name: vjepa2_1_vit_large_384
    load_from: facebookresearch/vjepa2
    source: github
    video_mode: true
    use_sdpa: true
    use_rope: true
    use_activation_checkpointing: true
    dtype: bfloat16
    patch_size: 16
    tubelet_size: 2
    crop_size: 384
    unload: true
    freeze: true
  predictor:
    language:
      model: meta-llama/Llama-3.2-1B
      num_layers: 4
      max_query_length: 512
      unload: false
      freeze: false
    compressor:
      type: qformer
      num_query_tokens: 512
      num_layers: 4
      num_heads: 8
      mlp_ratio: 4.0
      dropout: 0.2
      cross_attention_frequency: 1
      lr_multiplier: 2.0
  target:
    cosmos_dir: null
    reason1_ckpt: nvidia/Cosmos-Reason1-7B
    crossattn_proj_path: data/wts_cache/crossattn_proj.pt
    embedding_concat_strategy: full_concat
    n_layers_per_group: 5
    lr_multiplier: 0.05
    unload: true
    freeze: true
  decoder:
    model: null
    unload: true
    freeze: false
  runtime:
    trust_remote_code: true
    hf_token: false
    torch_dtype: auto
    tiny: false
  embedding_dim: 1024
  temperature: 0.1
data:
  train_file: data/vl_jepa/train.jsonl
  eval_file: data/vl_jepa/val.jsonl
  train_fraction: 1.0
  query_override: null
  target_type: short
  max_target_length: 512
training:
  runtime:
    seed: 42
    deterministic: true
    num_workers: 24
  checkpoint:
    init_checkpoint: null
    init_checkpoint_run_name: null
    require_init_checkpoint: false
    epochs: 10
    patience: 5
    min_delta: 0.0
    early_stop: true
  schedule:
    num_epochs: 50
  batching:
    batch_size: 60
    contrastive_accum_batches: 1
    grad_accum: 1
  optimization:
    type: adamw
    learning_rate: 4.0e-05
    weight_decay: 0.04
    max_grad_norm: 10.0
  scheduler:
    type: cosine
    min_learning_rate: 1.0e-06
    warmup_ratio: 0.05
  loss:
    label_smoothing: 0.05
    enabled: set_match:mse=1.0
    center: false
    target_mean_path: data/wts_cache/target_mean.pt
  ema:
    enabled: false
    decay:
    - 0.99
    - 0.9999
  logging:
    tensorboard: true
    log_steps: 10
    progress_type: tqdm
    save_csv: true
    save_batch_csv: true
    save_epoch_csv: true
    log_batch_scalars: true
  evaluation:
    epochs: 1
    max_batches: null
eval:
  pad_zero_threshold: 10.0  # seq-infer zeroes tokens with L2 norm below this (pad/null tail)