Update README.md
Browse files
README.md
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
def DummyTok(model_max_length=4):
|
| 2 |
import tempfile
|
| 3 |
|
|
@@ -13,4 +14,5 @@ def DummyTok(model_max_length=4):
|
|
| 13 |
)
|
| 14 |
|
| 15 |
real_tokenizer.save("dummy_tokenizer.json")
|
| 16 |
-
return real_tokenizer
|
|
|
|
|
|
| 1 |
+
```python
|
| 2 |
def DummyTok(model_max_length=4):
|
| 3 |
import tempfile
|
| 4 |
|
|
|
|
| 14 |
)
|
| 15 |
|
| 16 |
real_tokenizer.save("dummy_tokenizer.json")
|
| 17 |
+
return real_tokenizer
|
| 18 |
+
```
|