Mantis-50M / tokenizer_config.json

Mantis-50M: final checkpoint (step 34,000), model card, evals

0a91d5b verified 7 days ago

278 Bytes

{
  "backend": "tokenizers",
  "eos_token": "<|endoftext|>",
  "model_max_length": 1024,
  "pad_token": "<|endoftext|>",
  "tokenizer_class": "PreTrainedTokenizerFast",
  "clean_up_tokenization_spaces": false,
  "model_input_names": [
    "input_ids",
    "attention_mask"
  ]
}

	{
	"backend": "tokenizers",
	"eos_token": "<\|endoftext\|>",
	"model_max_length": 1024,
	"pad_token": "<\|endoftext\|>",
	"tokenizer_class": "PreTrainedTokenizerFast",
	"clean_up_tokenization_spaces": false,
	"model_input_names": [
	"input_ids",
	"attention_mask"
	]
	}