mega-small / tokenizer_config.json

Upload MEGA model

a0883c1 verified 12 days ago

260 Bytes

{
  "tokenizer_class": "MegaTokenizer",
  "unk_token": "<unk>",
  "bos_token": "<s>",
  "eos_token": "</s>",
  "pad_token": "<pad>",
  "modality_tokens": [
    "<image>",
    "<audio>",
    "<video>"
  ],
  "model_type": "unigram_or_bpe",
  "vocab_size": 800
}

	{
	"tokenizer_class": "MegaTokenizer",
	"unk_token": "<unk>",
	"bos_token": "<s>",
	"eos_token": "</s>",
	"pad_token": "<pad>",
	"modality_tokens": [
	"<image>",
	"<audio>",
	"<video>"
	],
	"model_type": "unigram_or_bpe",
	"vocab_size": 800
	}