Add tokenizer

.gitignore

@@ -0,0 +1,4 @@
+*.ipynb
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+.virtual_documents

tokenizer/added_tokens.json

@@ -0,0 +1 @@
+{"[SAFE]": 50257, "[OK]": 50259, "<eos>": 50272, "[PARTNER_M]": 50263, "[PARTNER_F]": 50264, "<speaker2>": 50270, "[PARTNER_N]": 50265, "[SELF_N]": 50262, "[UNSAFE]": 50258, "<speaker1>": 50269, "[SELF_M]": 50260, "[ABOUT_M]": 50266, "<bos>": 50271, "[SELF_F]": 50261, "[ABOUT_N]": 50268, "[ABOUT_F]": 50267}

tokenizer/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<bos>", "eos_token": "<eos>", "unk_token": "<|endoftext|>", "additional_special_tokens": ["[SAFE]", "[UNSAFE]", "[OK]", "[SELF_M]", "[SELF_F]", "[SELF_N]", "[PARTNER_M]", "[PARTNER_F]", "[PARTNER_N]", "[ABOUT_M]", "[ABOUT_F]", "[ABOUT_N]", "<speaker1>", "<speaker2>"]}

tokenizer/tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2-xl", "tokenizer_class": "GPT2Tokenizer"}