config.yaml

# Config for super vocab built from tokenizers: flexitok/bpe_arb_Arab_8000, flexitok/bpe_ces_Latn_8000, flexitok/bpe_ltr_cmn_Hani_8000_v2, flexitok/bpe_dan_Latn_8000, flexitok/bpe_deu_Latn_8000, flexitok/bpe_ell_Grek_8000, flexitok/bpe_fas_Arab_8000, flexitok/bpe_fra_Latn_8000, flexitok/bpe_fw_edu_8000, flexitok/bpe_hun_Latn_8000, flexitok/bpe_ind_Latn_8000, flexitok/bpe_ita_Latn_8000, flexitok/bpe_ltr_jpn_Jpan_8000_v2, flexitok/bpe_nld_Latn_8000, flexitok/bpe_pol_Latn_8000, flexitok/bpe_por_Latn_8000, flexitok/bpe_rus_Cyrl_8000, flexitok/bpe_spa_Latn_8000, flexitok/bpe_swe_Latn_8000, flexitok/bpe_ltr_tur_Latn_8000_v2, flexitok/bpe_vie_Latn_8000
## Training superset tokenizer with individual tokenizers trained on Fineweb-2-hq
# use with apps/main/configs/flexitok/llama_1b_base.yaml
##TODO: add router details, make sure name is correct
model:
  vocab_size: 122554
name: safe
dump_dir: /fsx/craffel/lingua_logs/safe
data:
  tokenizer:
    name: supertokenizer
    seed: 42
    superset_code_name: safe
    n_words: 122554
    tokenizers:
    - name: huggingface
      path: flexitok/bpe_arb_Arab_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_ces_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_ltr_cmn_Hani_8000_v2
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_dan_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_deu_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_ell_Grek_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_fas_Arab_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_fra_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_fw_edu_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_hun_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_ind_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_ita_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_ltr_jpn_Jpan_8000_v2
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_nld_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_pol_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_por_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_rus_Cyrl_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_spa_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_swe_Latn_8000
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_ltr_tur_Latn_8000_v2
      load_supermapping: true
    - name: huggingface
      path: flexitok/bpe_vie_Latn_8000
      load_supermapping: true
checkpoint:
  path: /fsx/craffel/lingua_logs/checkpoints/safe