JuanbaiAI
/

GlobalTokenizer

Model card Files Files and versions

laohur commited on Jan 28, 2024

Commit

d05ff08

·

verified ·

1 Parent(s): d3c381f

Update README.md

Files changed (1) hide show

README.md +2 -2

README.md CHANGED Viewed

@@ -8,7 +8,7 @@ from transformers import AutoModelForMaskedLM, AutoTokenizer
 model_path = "JuanbaiAI/GlobalTokenizer"
 tokenizer = AutoTokenizer.from_pretrained(model_path)
-print(tokenizer.tokenize("Hello, 世界"))
 ```
@@ -307,7 +307,7 @@ from transformers import AutoModelForMaskedLM, AutoTokenizer
 model_path = "JuanbaiAI/GlobalTokenizer"
 tokenizer = AutoTokenizer.from_pretrained(model_path)
-print(tokenizer.tokenize("Hello, 世界"))
 doc = [
     "                                                          ",

 model_path = "JuanbaiAI/GlobalTokenizer"
 tokenizer = AutoTokenizer.from_pretrained(model_path)
+print(tokenizer("Hello, 世界"))
 ```
 model_path = "JuanbaiAI/GlobalTokenizer"
 tokenizer = AutoTokenizer.from_pretrained(model_path)
+print(tokenizer("Hello, 世界"))
 doc = [
     "                                                          ",