Spaces:
Sleeping
Sleeping
Update babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py
Browse files
babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py
CHANGED
|
@@ -194,6 +194,8 @@ class ILTranslatorLLMOnly:
|
|
| 194 |
return text
|
| 195 |
|
| 196 |
def calc_token_count(self, text: str) -> int:
|
|
|
|
|
|
|
| 197 |
try:
|
| 198 |
return len(self.tokenizer.encode(text, disallowed_special=()))
|
| 199 |
except Exception:
|
|
@@ -601,7 +603,7 @@ class ILTranslatorLLMOnly:
|
|
| 601 |
continue
|
| 602 |
|
| 603 |
# Check minimum length - advance progress bar if filtered out
|
| 604 |
-
if len(paragraph.unicode) < self.translation_config.min_text_length:
|
| 605 |
if pbar:
|
| 606 |
pbar.advance(1)
|
| 607 |
continue
|
|
|
|
| 194 |
return text
|
| 195 |
|
| 196 |
def calc_token_count(self, text: str) -> int:
|
| 197 |
+
if text is None: # ADD THIS LINE
|
| 198 |
+
return 0 # ADD THIS LINE
|
| 199 |
try:
|
| 200 |
return len(self.tokenizer.encode(text, disallowed_special=()))
|
| 201 |
except Exception:
|
|
|
|
| 603 |
continue
|
| 604 |
|
| 605 |
# Check minimum length - advance progress bar if filtered out
|
| 606 |
+
if not paragraph.unicode or len(paragraph.unicode) < self.translation_config.min_text_length:
|
| 607 |
if pbar:
|
| 608 |
pbar.advance(1)
|
| 609 |
continue
|