Commit
·
dc71c13
1
Parent(s):
143f535
add comp back
Browse files- generic_ner.py +1 -1
generic_ner.py
CHANGED
|
@@ -87,7 +87,7 @@ WHITESPACE_RULES = {
|
|
| 87 |
}
|
| 88 |
|
| 89 |
|
| 90 |
-
def tokenize(text: str, language: str) -> list[str]:
|
| 91 |
"""Apply whitespace rules to the given text and language, separating it into tokens.
|
| 92 |
|
| 93 |
Args:
|
|
|
|
| 87 |
}
|
| 88 |
|
| 89 |
|
| 90 |
+
def tokenize(text: str, language: str = "other") -> list[str]:
|
| 91 |
"""Apply whitespace rules to the given text and language, separating it into tokens.
|
| 92 |
|
| 93 |
Args:
|