Spaces:
Sleeping
Sleeping
Update babeldoc/format/pdf/document_il/midend/il_translator.py
Browse files
babeldoc/format/pdf/document_il/midend/il_translator.py
CHANGED
|
@@ -417,6 +417,8 @@ class ILTranslator:
|
|
| 417 |
return text
|
| 418 |
|
| 419 |
def calc_token_count(self, text: str) -> int:
|
|
|
|
|
|
|
| 420 |
try:
|
| 421 |
return len(self.tokenizer.encode(text, disallowed_special=()))
|
| 422 |
except Exception:
|
|
|
|
| 417 |
return text
|
| 418 |
|
| 419 |
def calc_token_count(self, text: str) -> int:
|
| 420 |
+
if text is None: # ADD THIS LINE
|
| 421 |
+
return 0 # ADD THIS LINE
|
| 422 |
try:
|
| 423 |
return len(self.tokenizer.encode(text, disallowed_special=()))
|
| 424 |
except Exception:
|