{
  "config": {
    "run_name": "captionbert-8192-v2",
    "sources": [
      {
        "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
        "n_chunks": 66,
        "chunk_rows": 500000,
        "missing": {
          "modern": []
        }
      }
    ],
    "experts": [
      "bert",
      "modern",
      "roberta",
      "albert",
      "distil"
    ],
    "ref_expert": "bert",
    "ref_hf_name": "google-bert/bert-base-uncased",
    "require_all_experts": true,
    "caption_field": "(flat)",
    "caption_field_candidates": [
      "caption_llava",
      "caption",
      "caption_llava_short"
    ],
    "work_dir": "/content/cbv2",
    "keep_expert_shards": false,
    "disk_floor_gb": 25.0,
    "ram_resident": true,
    "preflight": true,
    "hf_repo": "AbstractPhil/captionbert-8192-v2",
    "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
    "push_targets": true,
    "hf_push": true,
    "push_every_min": 30.0,
    "keep_local_ckpts": 3,
    "parity_chunk": 0,
    "parity_n": 64,
    "parity_min_cos": 0.999,
    "fit_chunks": [
      0,
      11,
      22,
      33,
      44,
      55
    ],
    "fit_rows_per_chunk": 4000,
    "holdout_chunks": [
      60,
      61
    ],
    "fit_seed": 0,
    "d_model": 512,
    "n_heads": 8,
    "n_layers": 12,
    "d_ff": 2048,
    "max_len": 8192,
    "output_dim": 768,
    "dropout": 0.1,
    "pooling": "mean",
    "max_tokens": 256,
    "length_bucketing": true,
    "bucket_window": 64,
    "grad_checkpointing": true,
    "vram_probe": true,
    "epochs": 4,
    "batch_size": 2048,
    "lr": 0.0006,
    "min_lr": 1e-06,
    "warmup_steps": 2000,
    "grad_clip": 1.0,
    "seed": 42,
    "amp": true,
    "num_workers": 0,
    "log_every": 50,
    "eval_every": 1000,
    "ckpt_every_min": 20.0,
    "nce_weight": 1.0,
    "mse_weight": 1.0,
    "nce_temperature": 0.07,
    "cv_weight": 0.0,
    "cv_target": 0.084,
    "run_stage0": true,
    "run_stage1": true,
    "run_stage2": true,
    "run_stage3": true,
    "resume": true
  },
  "final": {
    "mimicry_r1": 0.9975000023841858,
    "cos_to_target": 0.8409875631332397,
    "self_cos": 0.003099770750850439,
    "erank": 102.90254876814863,
    "cv": 0.09320895820921411,
    "n": 5120,
    "frame_r1_after_rotation": 0.9984999895095825,
    "frame_cos_after_rotation": 0.8356035351753235,
    "frame_n_heldout": 2000
  }
}