Hugging Face's logo

AbstractPhil
/
captionbert-8192-v2

Feature Extraction
Transformers
TensorBoard
Safetensors
English
captionbert_v2
sentence-similarity
consensus-distillation
geometric-deep-learning
amoe
custom_code
Model card Files
xet
 Metrics Community
captionbert-8192-v2
File size: 2,387 Bytes
12f2835
 
 
 
 
 
 
 
 
ffc8b95
12f2835
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ffc8b95
12f2835
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
{
  "config": {
    "run_name": "captionbert-8192-v2",
    "sources": [
      {
        "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
        "n_chunks": 66,
        "chunk_rows": 500000,
        "missing": {
          "modern": []
        }
      }
    ],
    "experts": [
      "bert",
      "modern",
      "roberta",
      "albert",
      "distil"
    ],
    "ref_expert": "bert",
    "ref_hf_name": "google-bert/bert-base-uncased",
    "require_all_experts": true,
    "caption_field": "(flat)",
    "caption_field_candidates": [
      "caption_llava",
      "caption",
      "caption_llava_short"
    ],
    "work_dir": "/content/cbv2",
    "keep_expert_shards": false,
    "disk_floor_gb": 25.0,
    "ram_resident": true,
    "preflight": true,
    "hf_repo": "AbstractPhil/captionbert-8192-v2",
    "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
    "push_targets": true,
    "hf_push": true,
    "push_every_min": 30.0,
    "keep_local_ckpts": 3,
    "parity_chunk": 0,
    "parity_n": 64,
    "parity_min_cos": 0.999,
    "fit_chunks": [
      0,
      11,
      22,
      33,
      44,
      55
    ],
    "fit_rows_per_chunk": 4000,
    "holdout_chunks": [
      60,
      61
    ],
    "fit_seed": 0,
    "d_model": 512,
    "n_heads": 8,
    "n_layers": 12,
    "d_ff": 2048,
    "max_len": 8192,
    "output_dim": 768,
    "dropout": 0.1,
    "pooling": "mean",
    "max_tokens": 256,
    "length_bucketing": true,
    "bucket_window": 64,
    "grad_checkpointing": true,
    "vram_probe": true,
    "epochs": 4,
    "batch_size": 2048,
    "lr": 0.0006,
    "min_lr": 1e-06,
    "warmup_steps": 2000,
    "grad_clip": 1.0,
    "seed": 42,
    "amp": true,
    "num_workers": 0,
    "log_every": 50,
    "eval_every": 1000,
    "ckpt_every_min": 20.0,
    "nce_weight": 1.0,
    "mse_weight": 1.0,
    "nce_temperature": 0.07,
    "cv_weight": 0.0,
    "cv_target": 0.084,
    "run_stage0": true,
    "run_stage1": true,
    "run_stage2": true,
    "run_stage3": true,
    "resume": true
  },
  "final": {
    "mimicry_r1": 0.9975000023841858,
    "cos_to_target": 0.8409875631332397,
    "self_cos": 0.003099770750850439,
    "erank": 102.90254876814863,
    "cv": 0.09320895820921411,
    "n": 5120,
    "frame_r1_after_rotation": 0.9984999895095825,
    "frame_cos_after_rotation": 0.8356035351753235,
    "frame_n_heldout": 2000
  }
}