Update tokenizergen.py
Browse files- tokenizergen.py +1 -1
tokenizergen.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
from transformers import AutoTokenizer
|
| 2 |
|
| 3 |
# Download and save the tokenizer files
|
| 4 |
-
tokenizer = AutoTokenizer.from_pretrained("
|
| 5 |
tokenizer.save_pretrained("BuddAi") # Creates a folder with all tokenizer files
|
| 6 |
print("Tokenizer files saved to 'BuddAi/' directory!")
|
|
|
|
| 1 |
from transformers import AutoTokenizer
|
| 2 |
|
| 3 |
# Download and save the tokenizer files
|
| 4 |
+
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
|
| 5 |
tokenizer.save_pretrained("BuddAi") # Creates a folder with all tokenizer files
|
| 6 |
print("Tokenizer files saved to 'BuddAi/' directory!")
|