CaptMetal commited on
Commit
f92e6f0
·
verified ·
1 Parent(s): 25d28df

Update tokenizergen.py

Browse files
Files changed (1) hide show
  1. tokenizergen.py +1 -1
tokenizergen.py CHANGED
@@ -1,6 +1,6 @@
1
  from transformers import AutoTokenizer
2
 
3
  # Download and save the tokenizer files
4
- tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
5
  tokenizer.save_pretrained("BuddAi") # Creates a folder with all tokenizer files
6
  print("Tokenizer files saved to 'BuddAi/' directory!")
 
1
  from transformers import AutoTokenizer
2
 
3
  # Download and save the tokenizer files
4
+ tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
5
  tokenizer.save_pretrained("BuddAi") # Creates a folder with all tokenizer files
6
  print("Tokenizer files saved to 'BuddAi/' directory!")