Commit
·
927b82e
1
Parent(s):
bd6b737
Update README.md
Browse files
README.md
CHANGED
|
@@ -45,9 +45,8 @@ end_token = "<|"
|
|
| 45 |
tokenizer = GPT2Tokenizer.from_pretrained('gpt2-large')
|
| 46 |
model = GPT2LMHeadModel.from_pretrained('gpt2-large')
|
| 47 |
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
|
| 48 |
-
tokenizer.add_special_tokens({'eos_token': '<|End|>'})
|
| 49 |
special_tokens = {
|
| 50 |
-
"additional_special_tokens": ["<|USER|>", "<|
|
| 51 |
}
|
| 52 |
tokenizer.add_special_tokens(special_tokens)
|
| 53 |
model.resize_token_embeddings(len(tokenizer))
|
|
|
|
| 45 |
tokenizer = GPT2Tokenizer.from_pretrained('gpt2-large')
|
| 46 |
model = GPT2LMHeadModel.from_pretrained('gpt2-large')
|
| 47 |
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
|
|
|
|
| 48 |
special_tokens = {
|
| 49 |
+
"additional_special_tokens": ["<|USER|>", "<|ASSISTANT|>"]
|
| 50 |
}
|
| 51 |
tokenizer.add_special_tokens(special_tokens)
|
| 52 |
model.resize_token_embeddings(len(tokenizer))
|