amoe/collective/config.json

{
  "run_name": "captionbert-v2-collective",
  "trunk_repo": "AbstractPhil/captionbert-8192-v2",
  "trunk_ckpt": "checkpoints/best_model.pt",
  "tokenizer": "google-bert/bert-base-uncased",
  "d_model": 512,
  "n_heads": 8,
  "n_layers": 12,
  "d_ff": 2048,
  "max_len": 8192,
  "output_dim": 768,
  "pooling": "mean",
  "arms": [
    [
      "equiv",
      {
        "kind": "hub",
        "path": "amoe/moe/equiv.anchor.pt",
        "sources": [
          [
            "sentence-transformers/all-nli",
            "triplet",
            "anchor",
            "positive",
            "negative",
            200000
          ]
        ]
      }
    ],
    [
      "simplify",
      {
        "kind": "hub",
        "path": "amoe/moe/simplify.anchor.pt",
        "sources": [
          [
            "sentence-transformers/simple-wiki",
            "pair",
            "text",
            "simplified",
            null,
            0
          ],
          [
            "sentence-transformers/altlex",
            "pair",
            "text",
            "simplified",
            null,
            0
          ],
          [
            "sentence-transformers/sentence-compression",
            "pair",
            "text",
            "simplified",
            null,
            0
          ]
        ]
      }
    ],
    [
      "paraphrase",
      {
        "kind": "data",
        "sources": [
          [
            "sentence-transformers/quora-duplicates",
            "triplet",
            "anchor",
            "positive",
            "negative",
            0
          ]
        ]
      }
    ],
    [
      "lexical",
      {
        "kind": "wordnet"
      }
    ],
    [
      "topical",
      {
        "kind": "data",
        "sources": [
          [
            "sentence-transformers/specter",
            "triplet",
            "anchor",
            "positive",
            "negative",
            200000
          ]
        ]
      }
    ],
    [
      "random",
      {
        "kind": "random"
      }
    ]
  ],
  "wordnet_max": 156000,
  "dedup_jaccard": 0.95,
  "base_arms": [
    "equiv",
    "simplify"
  ],
  "greedy": true,
  "greedy_rounds": 4,
  "select_on": "STS-B",
  "n_slots": 16,
  "K": 64,
  "D": 4,
  "tau": 0.1,
  "hidden": 178,
  "gate_init": -3.0,
  "anchor_steps": 1500,
  "anchor_lr": 0.001,
  "batch_size": 256,
  "temperature": 0.05,
  "max_tokens": 64,
  "align_steps": 800,
  "align_lr": 0.001,
  "align_emb": 64,
  "check_every": 200,
  "usage_ppl_floor": 1.5,
  "usage_min": 0.02,
  "max_strikes": 3,
  "mix_rows": true,
  "align_taus": [
    0.1
  ],
  "align_seeds": [
    0
  ],
  "per_block_report": true,
  "seed": 0,
  "log_every": 100,
  "eval_every": 500,
  "reuse_anchors": true,
  "out_dir": "/content/amoe_collective",
  "hf_repo": "AbstractPhil/captionbert-8192-v2",
  "hf_path": "amoe/collective",
  "hf_private": false,
  "hf_push": true
}