Add usage examples

README.md

@@ -1,29 +1,3 @@
----
-license: apache-2.0
----
-
-Upstage `solar-1-mini` tokenizer
-- Vocab size: 64,000
-- Langauge support: English, Korean, Japanese and more
-
-Please use this tokenizer for tokenizing inputs for the Upstage [solar-1-mini-chat](https://developers.upstage.ai/docs/apis/chat) model.
-
-You can load it with the tokenizer library like this:
-
-```python
-from tokenizers import Tokenizer
-
-tokenizer = Tokenizer.from_pretrained("upstage/solar-1-mini-tokenizer")
-text = "Hi, how are you?"
-enc = tokenizer.encode(text)
-print("Encoded input:")
-print(enc)
-
-inv_vocab = {v: k for k, v in tokenizer.get_vocab().items()}
-tokens = [inv_vocab[token_id] for token_id in enc.ids]
-print("Tokens:")
-print(tokens)
-
-number_of_tokens = len(enc.ids)
-print("Number of tokens:", number_of_tokens)
-```
+---
+license: apache-2.0
+---

tokenizer_config.json

@@ -1,140 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "add_prefix_space": null,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<|startoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63988": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63989": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63990": {
-      "content": "<|reserved_0|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63991": {
-      "content": "<|reserved_1|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63992": {
-      "content": "<|reserved_2|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63993": {
-      "content": "<|reserved_3|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63994": {
-      "content": "<|reserved_4|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63995": {
-      "content": "<|reserved_5|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63996": {
-      "content": "<|reserved_6|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63997": {
-      "content": "<|reserved_7|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63998": {
-      "content": "<|reserved_8|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "63999": {
-      "content": "<|reserved_9|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<|startoftext|>",
-  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
-  "extra_special_tokens": {},
-  "legacy": true,
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|endoftext|>",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}