delete tokenizer

app.py

@@ -821,8 +821,9 @@ if __name__ == "__main__":
 
     personality_prompt = build_personality_prompt(personality)
     system_instruction_without_context = SYSTEM_INSTRUCTION + "\n" + personality_prompt + "\n"
-    tokenizer = AutoTokenizer.from_pretrained(RESPONSE_GENERATOR)
-    tokenizer, terminator = prepare_tokenizer(tokenizer)
+    # tokenizer = AutoTokenizer.from_pretrained(RESPONSE_GENERATOR)
+    # tokenizer, terminator = prepare_tokenizer(tokenizer)
+    terminator = ["<eos>", "<unk>", "<sep>", "<pad>", "<cls>", "<mask>"]
     print(f"Terminator: {terminator}")
     demo = create_demo(terminator, system_instruction_without_context, stock_context_list, raw_context_list)
     demo.launch(share=True)