Update app.py
Browse files
app.py
CHANGED
|
@@ -15,9 +15,9 @@ def get_distinct_colors(n):
|
|
| 15 |
colors.append(color)
|
| 16 |
return colors
|
| 17 |
|
| 18 |
-
def tokenize_text(hf_model_id, text,
|
| 19 |
try:
|
| 20 |
-
tokenizer = AutoTokenizer.from_pretrained(hf_model_id, token
|
| 21 |
tokens = tokenizer.tokenize(text)
|
| 22 |
token_count = len(tokens)
|
| 23 |
colors = get_distinct_colors(token_count)
|
|
|
|
| 15 |
colors.append(color)
|
| 16 |
return colors
|
| 17 |
|
| 18 |
+
def tokenize_text(hf_model_id, text, token=None):
|
| 19 |
try:
|
| 20 |
+
tokenizer = AutoTokenizer.from_pretrained(hf_model_id, access_token=token)
|
| 21 |
tokens = tokenizer.tokenize(text)
|
| 22 |
token_count = len(tokens)
|
| 23 |
colors = get_distinct_colors(token_count)
|