File size: 1,969 Bytes
4d2a76c 9d933c2 4d2a76c 9d933c2 4d2a76c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
# Config for super vocab built from tokenizers: flexitok/bpe_script_Arab_16000, flexitok/bpe_script_CmJp_16000, flexitok/bpe_ltr_ell_Grek_8000_v2, flexitok/bpe_ltr_fw_edu_32000_v2, flexitok/bpe_ltr_hun_Latn_8000_v2, flexitok/bpe_ltr_rus_Cyrl_16000_v2, flexitok/bpe_ltr_tur_Latn_8000_v2, flexitok/bpe_script_Germ_32000, flexitok/bpe_script_Roma_32000, flexitok/bpe_script_SEAS_16000, flexitok/bpe_script_Slav_16000
## Training superset tokenizer with individual tokenizers trained on Fineweb-2-hq
# use with apps/main/configs/flexitok/llama_1b_base.yaml
##TODO: add router details, make sure name is correct
model:
vocab_size: 165022
name: script_1
dump_dir: /fsx/craffel/lingua_logs/script_1
data:
tokenizer:
name: supertokenizer
seed: 42
superset_code_name: script_1
n_words: 165022
tokenizers:
- name: huggingface
path: flexitok/bpe_script_Arab_16000
load_supermapping: true
- name: huggingface
path: flexitok/bpe_script_CmJp_16000
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_ell_Grek_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_fw_edu_32000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_hun_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_rus_Cyrl_16000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_tur_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_script_Germ_32000
load_supermapping: true
- name: huggingface
path: flexitok/bpe_script_Roma_32000
load_supermapping: true
- name: huggingface
path: flexitok/bpe_script_SEAS_16000
load_supermapping: true
- name: huggingface
path: flexitok/bpe_script_Slav_16000
load_supermapping: true
checkpoint:
path: /fsx/craffel/lingua_logs/checkpoints/script_1
|