Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
|
@@ -9,7 +9,7 @@ models = ["bigcode/tokenizer", "facebook/incoder-6B", "Salesforce/codegen-16B-mo
|
|
| 9 |
@st.cache()
|
| 10 |
def load_tokenizer():
|
| 11 |
if selected_model == "bigcode/tokenizer":
|
| 12 |
-
tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-
|
| 13 |
else:
|
| 14 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
| 15 |
return tokenizer
|
|
|
|
| 9 |
@st.cache()
|
| 10 |
def load_tokenizer():
|
| 11 |
if selected_model == "bigcode/tokenizer":
|
| 12 |
+
tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-punctuation-bytelevel-bpe-py-js-java-50k")
|
| 13 |
else:
|
| 14 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
| 15 |
return tokenizer
|