Update README.md
Browse files
README.md
CHANGED
|
@@ -41,7 +41,7 @@ pip install -U tokenizers transformers huggingface_hub
|
|
| 41 |
|
| 42 |
## Usage
|
| 43 |
|
| 44 |
-
Replace `
|
| 45 |
|
| 46 |
### Transformers
|
| 47 |
|
|
@@ -49,7 +49,7 @@ Replace `SamAI/SamAI-Tokenizer` with the published repository ID.
|
|
| 49 |
from transformers import PreTrainedTokenizerFast
|
| 50 |
|
| 51 |
tokenizer = PreTrainedTokenizerFast.from_pretrained(
|
| 52 |
-
"
|
| 53 |
)
|
| 54 |
|
| 55 |
text = "المملكة العربية السعودية"
|
|
@@ -67,7 +67,7 @@ from huggingface_hub import hf_hub_download
|
|
| 67 |
from tokenizers import Tokenizer
|
| 68 |
|
| 69 |
path = hf_hub_download(
|
| 70 |
-
repo_id="
|
| 71 |
filename="tokenizer.json",
|
| 72 |
)
|
| 73 |
tokenizer = Tokenizer.from_file(path)
|
|
|
|
| 41 |
|
| 42 |
## Usage
|
| 43 |
|
| 44 |
+
Replace `Samai-sa/SamAI-Tokenizer` with the published repository ID.
|
| 45 |
|
| 46 |
### Transformers
|
| 47 |
|
|
|
|
| 49 |
from transformers import PreTrainedTokenizerFast
|
| 50 |
|
| 51 |
tokenizer = PreTrainedTokenizerFast.from_pretrained(
|
| 52 |
+
"Samai-sa/SamAI-Tokenizer"
|
| 53 |
)
|
| 54 |
|
| 55 |
text = "المملكة العربية السعودية"
|
|
|
|
| 67 |
from tokenizers import Tokenizer
|
| 68 |
|
| 69 |
path = hf_hub_download(
|
| 70 |
+
repo_id="Samai-sa/SamAI-Tokenizer",
|
| 71 |
filename="tokenizer.json",
|
| 72 |
)
|
| 73 |
tokenizer = Tokenizer.from_file(path)
|