dnzblgn commited on
Commit
6e2e192
·
verified ·
1 Parent(s): 7d7f8be

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -5
app.py CHANGED
@@ -5,12 +5,13 @@ from transformers import PreTrainedTokenizerFast
5
 
6
  # Load all tokenizers (at startup)
7
  tokenizers = {
8
- "GPT-2 Tokenizer": AutoTokenizer.from_pretrained("gpt2"),
9
- "RoBERTa Tokenizer": AutoTokenizer.from_pretrained("roberta-base"),
10
- "DistilGPT-2 Tokenizer": AutoTokenizer.from_pretrained("distilgpt2"),
 
11
  "bert-base-german-cased Tokenizer": AutoTokenizer.from_pretrained("bert-base-german-cased"),
12
- "DeepSeek Tokenizer": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1"),
13
- "Tabularis German Tokenizer": PreTrainedTokenizerFast(tokenizer_file="tokenizer.json")
14
  }
15
 
16
  # Fancy token visualization with random colors
 
5
 
6
  # Load all tokenizers (at startup)
7
  tokenizers = {
8
+ "Tabularis German Tokenizer": PreTrainedTokenizerFast(tokenizer_file="tokenizer.json"),
9
+ "KoichiYasuoka/bert-base-german-upos": AutoTokenizer.from_pretrained("KoichiYasuoka/bert-base-german-upos"),
10
+ "benjamin/gerpt2-large": AutoTokenizer.from_pretrained("benjamin/gerpt2-large"),
11
+ "deepset/gbert-base": AutoTokenizer.from_pretrained("deepset/gbert-base"),
12
  "bert-base-german-cased Tokenizer": AutoTokenizer.from_pretrained("bert-base-german-cased"),
13
+ "MiriUll/gpt2-wechsel-german_easy": AutoTokenizer.from_pretrained("MiriUll/gpt2-wechsel-german_easy"),
14
+ "DeepSeek Tokenizer": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1")
15
  }
16
 
17
  # Fancy token visualization with random colors