Hugging Face's logo

AbstractPhil
/
captionbert-8192-v2

Feature Extraction
Transformers
TensorBoard
Safetensors
English
captionbert_v2
sentence-similarity
consensus-distillation
geometric-deep-learning
amoe
custom_code
Model card Files
xet
 Metrics Community
captionbert-8192-v2
File size: 1,844 Bytes
9915609
 
 
 
 
 
 
 
468ad1b
9915609
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0f36950
 
 
 
9915609
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
{
  "run_name": "captionbert-8192-v2",
  "sources": [
    {
      "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
      "n_chunks": 66,
      "chunk_rows": 500000,
      "missing": {
        "modern": []
      }
    }
  ],
  "experts": [
    "bert",
    "modern",
    "roberta",
    "albert",
    "distil"
  ],
  "ref_expert": "bert",
  "ref_hf_name": "google-bert/bert-base-uncased",
  "require_all_experts": true,
  "caption_field": "(flat)",
  "caption_field_candidates": [
    "caption_llava",
    "caption",
    "caption_llava_short"
  ],
  "work_dir": "/content/cbv2",
  "keep_expert_shards": false,
  "disk_floor_gb": 25.0,
  "ram_resident": true,
  "preflight": true,
  "hf_repo": "AbstractPhil/captionbert-8192-v2",
  "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
  "push_targets": true,
  "hf_push": true,
  "push_every_min": 30.0,
  "keep_local_ckpts": 3,
  "parity_chunk": 0,
  "parity_n": 64,
  "parity_min_cos": 0.999,
  "fit_chunks": [
    0,
    11,
    22,
    33,
    44,
    55
  ],
  "fit_rows_per_chunk": 4000,
  "holdout_chunks": [
    60,
    61
  ],
  "fit_seed": 0,
  "d_model": 512,
  "n_heads": 8,
  "n_layers": 12,
  "d_ff": 2048,
  "max_len": 8192,
  "output_dim": 768,
  "dropout": 0.1,
  "pooling": "mean",
  "max_tokens": 256,
  "length_bucketing": true,
  "bucket_window": 64,
  "grad_checkpointing": true,
  "vram_probe": true,
  "epochs": 4,
  "batch_size": 2048,
  "lr": 0.0006,
  "min_lr": 1e-06,
  "warmup_steps": 2000,
  "grad_clip": 1.0,
  "seed": 42,
  "amp": true,
  "num_workers": 0,
  "log_every": 50,
  "eval_every": 1000,
  "ckpt_every_min": 20.0,
  "nce_weight": 1.0,
  "mse_weight": 1.0,
  "nce_temperature": 0.07,
  "cv_weight": 0.0,
  "cv_target": 0.084,
  "run_stage0": true,
  "run_stage1": true,
  "run_stage2": true,
  "run_stage3": true,
  "resume": true
}