Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
|
@@ -5,12 +5,13 @@ from transformers import PreTrainedTokenizerFast
|
|
| 5 |
|
| 6 |
# Load all tokenizers (at startup)
|
| 7 |
tokenizers = {
|
| 8 |
-
"
|
| 9 |
-
"
|
| 10 |
-
"
|
|
|
|
| 11 |
"bert-base-german-cased Tokenizer": AutoTokenizer.from_pretrained("bert-base-german-cased"),
|
| 12 |
-
"
|
| 13 |
-
"
|
| 14 |
}
|
| 15 |
|
| 16 |
# Fancy token visualization with random colors
|
|
|
|
| 5 |
|
| 6 |
# Load all tokenizers (at startup)
|
| 7 |
tokenizers = {
|
| 8 |
+
"Tabularis German Tokenizer": PreTrainedTokenizerFast(tokenizer_file="tokenizer.json"),
|
| 9 |
+
"KoichiYasuoka/bert-base-german-upos": AutoTokenizer.from_pretrained("KoichiYasuoka/bert-base-german-upos"),
|
| 10 |
+
"benjamin/gerpt2-large": AutoTokenizer.from_pretrained("benjamin/gerpt2-large"),
|
| 11 |
+
"deepset/gbert-base": AutoTokenizer.from_pretrained("deepset/gbert-base"),
|
| 12 |
"bert-base-german-cased Tokenizer": AutoTokenizer.from_pretrained("bert-base-german-cased"),
|
| 13 |
+
"MiriUll/gpt2-wechsel-german_easy": AutoTokenizer.from_pretrained("MiriUll/gpt2-wechsel-german_easy"),
|
| 14 |
+
"DeepSeek Tokenizer": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1")
|
| 15 |
}
|
| 16 |
|
| 17 |
# Fancy token visualization with random colors
|