flexitok/unigram_fw_edu_16000
Updated
Fineweb2-HQ-Tokenizers
-
-
flexitok/bpe_vie_Latn_16000
Updated -
flexitok/bpe_ita_Latn_16000
Updated -
flexitok/bpe_rus_Cyrl_16000
Updated -
flexitok/bpe_arb_Arab_16000
Updated -
flexitok/bpe_deu_Latn_16000
Updated -
flexitok/unigram_ell_Grek_16000
Updated -
flexitok/bpe_hun_Latn_16000
Updated -
flexitok/bpe_fas_Arab_16000
Updated -
flexitok/unigram_jpn_Jpan_64000
Updated -
flexitok/unigram_hun_Latn_16000
Updated -
flexitok/bpe_tur_Latn_64000
Updated -
flexitok/unigram_por_Latn_16000
Updated -
flexitok/bpe_cmn_Hani_32000
Updated -
flexitok/bpe_spa_Latn_32000
Updated -
flexitok/bpe_por_Latn_32000
Updated -
flexitok/bpe_ell_Grek_16000
Updated -
flexitok/unigram_ces_Latn_128000
Updated -
flexitok/bpe_ind_Latn_128000
Updated -
flexitok/bpe_tur_Latn_16000
Updated -
flexitok/unigram_tur_Latn_128000
Updated -
flexitok/bpe_pol_Latn_32000
Updated -
flexitok/bpe_dan_Latn_128000
Updated -
flexitok/unigram_dan_Latn_32000
Updated -
flexitok/bpe_rus_Cyrl_128000
Updated -
flexitok/bpe_arb_Arab_32000
Updated -
flexitok/unigram_arb_Arab_16000
Updated -
flexitok/bpe_swe_Latn_16000
Updated -
flexitok/bpe_ces_Latn_128000
Updated -
flexitok/bpe_dan_Latn_64000
Updated -
flexitok/bpe_hun_Latn_32000
Updated -
flexitok/unigram_ell_Grek_128000
Updated -
flexitok/unigram_nld_Latn_64000
Updated -
flexitok/bpe_ita_Latn_128000
Updated -
flexitok/bpe_jpn_Jpan_128000
Updated -
flexitok/bpe_fw_edu_128000
Updated -
flexitok/unigram_dan_Latn_64000
Updated -
flexitok/bpe_arb_Arab_128000
Updated -
flexitok/unigram_ces_Latn_16000
Updated -
flexitok/unigram_ell_Grek_64000
Updated -
flexitok/bpe_por_Latn_64000
Updated -
flexitok/unigram_fra_Latn_64000
Updated -
flexitok/unigram_spa_Latn_64000
Updated -
flexitok/unigram_fra_Latn_16000
Updated -
flexitok/unigram_ces_Latn_32000
Updated -
flexitok/unigram_vie_Latn_128000
Updated -
flexitok/unigram_jpn_Jpan_128000
Updated -
flexitok/unigram_pol_Latn_64000
Updated -
flexitok/unigram_fra_Latn_128000
Updated -
flexitok/unigram_ita_Latn_16000
Updated -
flexitok/bpe_rus_Cyrl_32000
Updated -
flexitok/bpe_nld_Latn_128000
Updated -
flexitok/bpe_deu_Latn_128000
Updated -
flexitok/unigram_jpn_Jpan_16000
Updated -
flexitok/unigram_hun_Latn_64000
Updated -
flexitok/bpe_spa_Latn_64000
Updated -
flexitok/bpe_spa_Latn_128000
Updated -
flexitok/unigram_ind_Latn_32000
Updated -
flexitok/unigram_pol_Latn_128000
Updated -
flexitok/unigram_fas_Arab_64000
Updated -
flexitok/bpe_rus_Cyrl_64000
Updated -
flexitok/bpe_ind_Latn_64000
Updated -
flexitok/unigram_spa_Latn_16000
Updated -
flexitok/unigram_arb_Arab_64000
Updated -
flexitok/unigram_ita_Latn_128000
Updated -
flexitok/bpe_jpn_Jpan_32000
Updated -
flexitok/bpe_deu_Latn_64000
Updated -
flexitok/unigram_por_Latn_32000
Updated -
flexitok/unigram_ind_Latn_16000
Updated -
flexitok/bpe_cmn_Hani_64000
Updated -
flexitok/unigram_ces_Latn_64000
Updated -
flexitok/bpe_pol_Latn_128000
Updated -
flexitok/bpe_vie_Latn_64000
Updated -
flexitok/bpe_swe_Latn_128000
Updated -
flexitok/unigram_pol_Latn_32000
Updated -
flexitok/unigram_fw_edu_128000
Updated -
flexitok/unigram_deu_Latn_32000
Updated -
flexitok/unigram_por_Latn_128000
Updated -
flexitok/bpe_tur_Latn_32000
Updated -
flexitok/unigram_nld_Latn_16000
Updated -
flexitok/unigram_rus_Cyrl_32000
Updated -
flexitok/unigram_arb_Arab_128000
Updated -
flexitok/bpe_pol_Latn_64000
Updated -
flexitok/bpe_jpn_Jpan_64000
Updated -
flexitok/unigram_dan_Latn_128000
Updated -
flexitok/unigram_dan_Latn_16000
Updated -
flexitok/unigram_swe_Latn_32000
Updated -
flexitok/bpe_dan_Latn_32000
Updated -
flexitok/bpe_nld_Latn_32000
Updated -
flexitok/unigram_deu_Latn_64000
Updated -
flexitok/bpe_vie_Latn_128000
Updated -
flexitok/unigram_cmn_Hani_128000
Updated -
flexitok/unigram_jpn_Jpan_32000
Updated -
flexitok/unigram_ita_Latn_64000
Updated -
flexitok/unigram_tur_Latn_64000
Updated -
flexitok/bpe_fas_Arab_64000
Updated -
flexitok/unigram_por_Latn_64000
Updated -
flexitok/bpe_tur_Latn_128000
Updated -
flexitok/bpe_por_Latn_128000
Updated -
flexitok/bpe_fra_Latn_64000
Updated -
flexitok/bpe_fas_Arab_128000
Updated -
flexitok/bpe_hun_Latn_64000
Updated -
flexitok/bpe_ces_Latn_64000
Updated -
flexitok/unigram_ind_Latn_64000
Updated -
flexitok/unigram_ita_Latn_32000
Updated -
flexitok/bpe_dan_Latn_16000
Updated -
flexitok/bpe_ind_Latn_32000
Updated -
flexitok/unigram_spa_Latn_128000
Updated -
flexitok/unigram_tur_Latn_16000
Updated -
flexitok/unigram_hun_Latn_128000
Updated -
flexitok/bpe_arb_Arab_64000
Updated -
flexitok/bpe_ell_Grek_64000
Updated -
flexitok/bpe_ita_Latn_64000
Updated -
flexitok/unigram_fw_edu_64000
Updated -
flexitok/unigram_fw_edu_32000
Updated -
flexitok/unigram_rus_Cyrl_16000
Updated -
flexitok/unigram_spa_Latn_32000
Updated -
flexitok/bpe_pol_Latn_16000
Updated -
flexitok/unigram_ell_Grek_32000
Updated -
flexitok/bpe_fra_Latn_32000
Updated -
flexitok/unigram_fra_Latn_32000
Updated -
flexitok/unigram_cmn_Hani_32000
Updated -
flexitok/bpe_hun_Latn_128000
Updated -
flexitok/unigram_swe_Latn_64000
Updated -
flexitok/bpe_ell_Grek_32000
Updated -
flexitok/bpe_deu_Latn_32000
Updated -
flexitok/unigram_arb_Arab_32000
Updated -
flexitok/unigram_pol_Latn_16000
Updated -
flexitok/bpe_fra_Latn_128000
Updated -
flexitok/unigram_swe_Latn_128000
Updated -
flexitok/bpe_swe_Latn_32000
Updated -
flexitok/unigram_nld_Latn_32000
Updated -
flexitok/bpe_jpn_Jpan_16000
Updated -
flexitok/bpe_cmn_Hani_16000
Updated -
flexitok/bpe_spa_Latn_16000
Updated -
flexitok/bpe_fra_Latn_16000
Updated -
flexitok/bpe_ces_Latn_16000
Updated -
flexitok/unigram_vie_Latn_16000
Updated -
flexitok/unigram_hun_Latn_32000
Updated -
flexitok/bpe_nld_Latn_16000
Updated -
flexitok/unigram_rus_Cyrl_128000
Updated -
flexitok/unigram_deu_Latn_16000
Updated -
flexitok/bpe_fw_edu_64000
Updated -
flexitok/unigram_fas_Arab_32000
Updated -
flexitok/unigram_fas_Arab_128000
Updated -
flexitok/unigram_tur_Latn_32000
Updated -
flexitok/bpe_ell_Grek_128000
Updated -
flexitok/bpe_swe_Latn_64000
Updated -
flexitok/unigram_vie_Latn_32000
Updated -
flexitok/bpe_vie_Latn_32000
Updated -
flexitok/unigram_swe_Latn_16000
Updated -
flexitok/bpe_nld_Latn_64000
Updated -
flexitok/bpe_por_Latn_16000
Updated -
flexitok/bpe_fw_edu_16000
Updated -
flexitok/bpe_cmn_Hani_128000
Updated -
flexitok/unigram_ind_Latn_128000
Updated -
flexitok/bpe_ita_Latn_32000
Updated -
flexitok/unigram_rus_Cyrl_64000
Updated -
flexitok/bpe_dan_Latn_8000
Updated -
flexitok/bpe_fas_Arab_8000
Updated -
flexitok/bpe_fas_Arab_32000
Updated -
flexitok/bpe_hun_Latn_8000
Updated -
flexitok/bpe_ell_Grek_8000
Updated -
flexitok/bpe_ces_Latn_8000
Updated -
flexitok/bpe_arb_Arab_8000
Updated -
flexitok/bpe_ind_Latn_8000
Updated -
flexitok/bpe_pol_Latn_8000
Updated -
flexitok/bpe_tur_Latn_8000
Updated -
flexitok/bpe_por_Latn_8000
Updated -
flexitok/bpe_ita_Latn_8000
Updated -
flexitok/unigram_ces_Latn_8000
Updated -
flexitok/unigram_vie_Latn_8000
Updated -
flexitok/unigram_swe_Latn_8000
Updated -
flexitok/bpe_nld_Latn_8000
Updated -
flexitok/unigram_arb_Arab_8000
Updated -
flexitok/unigram_dan_Latn_8000
Updated -
flexitok/unigram_tur_Latn_8000
Updated -
flexitok/bpe_vie_Latn_8000
Updated -
flexitok/unigram_hun_Latn_8000
Updated -
flexitok/unigram_por_Latn_8000
Updated -
flexitok/bpe_fra_Latn_8000
Updated -
flexitok/bpe_spa_Latn_8000
Updated -
flexitok/unigram_ind_Latn_8000
Updated -
flexitok/unigram_vie_Latn_64000
Updated -
flexitok/bpe_swe_Latn_8000
Updated -
flexitok/bpe_ces_Latn_32000
Updated -
flexitok/unigram_fas_Arab_8000
Updated -
flexitok/bpe_deu_Latn_8000
Updated -
flexitok/bpe_rus_Cyrl_8000
Updated -
flexitok/bpe_fw_edu_8000
Updated