Update create_granular_chunks.py

create_granular_chunks.py

@@ -1,3 +1,4 @@
+# create_granular_chunks.py
 import os
 import json
 import re
@@ -6,6 +7,7 @@ import nltk
 
 # Download punkt tokenizer if not already done (Ensure this runs once in your environment setup)
 nltk.download('punkt')
+nltk.download('punkt_tab')  # Also download punkt_tab to avoid LookupError
 
 # --- Configuration ---
 INPUT_FILE = "combined_context.jsonl"
@@ -112,7 +114,8 @@ def split_text_into_chunks(text: str, max_char_length: int = 1500, overlap: int
     if len(text) <= max_char_length:
         return [text]
 
-    sentences = nltk.sent_tokenize(text)
+    # Explicitly specify language to avoid punkt_tab error
+    sentences = nltk.tokenize.sent_tokenize(text, language='english')
     chunks = []
     current_chunk = ""