Improve text splitting and error handling in summarizer

summarizer.py

@@ -114,24 +114,45 @@ class DocumentSummarizer:
     def _split_text(self, text: str) -> List[str]:
         """텍스트를 적절한 크기로 분할"""
         try:
-            sentences = nltk.sent_tokenize(text)
+            # Use the configured tokenizer (either punkt or sent_tokenize)
+            if hasattr(self, 'tokenizer') and callable(self.tokenizer):
+                if self.tokenizer == nltk.tokenize.sent_tokenize:
+                    sentences = nltk.tokenize.sent_tokenize(text)
+                else:
+                    # Handle the case where tokenizer is a PunktSentenceTokenizer instance
+                    sentences = self.tokenizer.tokenize(text)
+            else:
+                # Fallback to default sentence tokenizer
+                nltk.download('punkt')
+                sentences = nltk.tokenize.sent_tokenize(text)
+            
             chunks = []
             current_chunk = ""
             
             for sentence in sentences:
-                if len(current_chunk) + len(sentence) <= self.chunk_size:
-                    current_chunk += " " + sentence
+                if len(current_chunk.split()) + len(sentence.split()) <= self.chunk_size:
+                    current_chunk = f"{current_chunk} {sentence}".strip()
                 else:
-                    chunks.append(current_chunk.strip())
+                    if current_chunk:  # Only add non-empty chunks
+                        chunks.append(current_chunk)
                     current_chunk = sentence
             
+            # Add the last chunk if it's not empty
             if current_chunk:
                 chunks.append(current_chunk.strip())
             
-            return chunks
+            return chunks if chunks else [text]  # Return at least one chunk
             
+        except LookupError as e:
+            # If punkt data is missing, try to download it
+            print(f"NLTK data missing, attempting to download: {e}")
+            nltk.download('punkt')
+            # Retry with the default tokenizer
+            return self._split_text(text)
         except Exception as e:
-            raise Exception(f"텍스트 분할 중 오류 발생: {str(e)}")
+            print(f"Error in _split_text: {str(e)}")
+            # If all else fails, return the original text as a single chunk
+            return [text]
 
 # 싱글톤 인스턴스 생성
 document_summarizer = DocumentSummarizer()