config/config.json

{
  "profile": "_applied",
  "run_name": "captionbert-8192-b",
  "hf_repo": "AbstractPhil/captionbert-8192-v2-B",
  "sources": [
    {
      "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
      "n_chunks": 66,
      "chunk_rows": 500000,
      "missing": {}
    }
  ],
  "experts": [
    "bert",
    "modern",
    "roberta",
    "albert",
    "distil"
  ],
  "ref_expert": "bert",
  "ref_hf_name": "google-bert/bert-base-uncased",
  "require_all_experts": true,
  "caption_field": "caption_llava",
  "caption_field_candidates": [
    "caption_llava",
    "caption",
    "caption_llava_short"
  ],
  "work_dir": "/content/cbv2",
  "keep_expert_shards": false,
  "disk_floor_gb": 25.0,
  "ram_resident": true,
  "preflight": true,
  "artifact_repo": "AbstractPhil/captionbert-8192-v2",
  "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
  "push_targets": true,
  "hf_push": true,
  "push_every_min": 30.0,
  "keep_local_ckpts": 3,
  "parity_chunk": 0,
  "parity_n": 64,
  "parity_min_cos": 0.999,
  "fit_chunks": [
    0,
    11,
    22,
    33,
    44,
    55
  ],
  "fit_rows_per_chunk": 4000,
  "holdout_chunks": [
    60,
    61
  ],
  "fit_seed": 0,
  "d_model": 512,
  "n_heads": 8,
  "n_layers": 12,
  "d_ff": 2048,
  "max_len": 8192,
  "output_dim": 768,
  "dropout": 0.1,
  "pooling": "mean",
  "max_tokens": 256,
  "length_bucketing": true,
  "bucket_window": 64,
  "grad_checkpointing": true,
  "vram_probe": true,
  "epochs": 4,
  "batch_size": 2048,
  "lr": 0.0006,
  "min_lr": 1e-06,
  "warmup_steps": 2000,
  "grad_clip": 1.0,
  "seed": 42,
  "amp": true,
  "num_workers": 0,
  "log_every": 50,
  "eval_every": 1000,
  "ckpt_every_min": 20.0,
  "fresh_start": true,
  "allow_mismatched_resume": false,
  "nce_weight": 1.0,
  "mse_weight": 1.0,
  "nce_temperature": 0.07,
  "cv_weight": 0.0,
  "cv_target": 0.084,
  "run_stage0": false,
  "run_stage1": false,
  "run_stage2": false,
  "run_stage3": true,
  "resume": true
}