Commit ·
1ef73c6
1
Parent(s): c5c3b3e
Upload folder using huggingface_hub
Browse files
README.md
CHANGED
|
@@ -41,7 +41,7 @@ import ctranslate2
|
|
| 41 |
import transformers
|
| 42 |
|
| 43 |
generator = ctranslate2.Generator("limcheekin/mpt-7b-instruct-ct2")
|
| 44 |
-
tokenizer = transformers.AutoTokenizer.from_pretrained("
|
| 45 |
|
| 46 |
prompt = "Long long time ago, "
|
| 47 |
tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt))
|
|
|
|
| 41 |
import transformers
|
| 42 |
|
| 43 |
generator = ctranslate2.Generator("limcheekin/mpt-7b-instruct-ct2")
|
| 44 |
+
tokenizer = transformers.AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
|
| 45 |
|
| 46 |
prompt = "Long long time ago, "
|
| 47 |
tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt))
|