Taha493 commited on
Commit
d61c156
·
verified ·
1 Parent(s): f803922

Update babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py

Browse files
babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py CHANGED
@@ -194,6 +194,8 @@ class ILTranslatorLLMOnly:
194
  return text
195
 
196
  def calc_token_count(self, text: str) -> int:
 
 
197
  try:
198
  return len(self.tokenizer.encode(text, disallowed_special=()))
199
  except Exception:
@@ -601,7 +603,7 @@ class ILTranslatorLLMOnly:
601
  continue
602
 
603
  # Check minimum length - advance progress bar if filtered out
604
- if len(paragraph.unicode) < self.translation_config.min_text_length:
605
  if pbar:
606
  pbar.advance(1)
607
  continue
 
194
  return text
195
 
196
  def calc_token_count(self, text: str) -> int:
197
+ if text is None: # ADD THIS LINE
198
+ return 0 # ADD THIS LINE
199
  try:
200
  return len(self.tokenizer.encode(text, disallowed_special=()))
201
  except Exception:
 
603
  continue
604
 
605
  # Check minimum length - advance progress bar if filtered out
606
+ if not paragraph.unicode or len(paragraph.unicode) < self.translation_config.min_text_length:
607
  if pbar:
608
  pbar.advance(1)
609
  continue