Update src/streamlit_app.py

src/streamlit_app.py

@@ -2,15 +2,8 @@ import streamlit as st
 import pyphen
 import re
 from typing import List, Tuple
-import nltk
-from nltk.tokenize import word_tokenize
 import string
-
-# Download required NLTK data
-try:
-    nltk.data.find('tokenizers/punkt')
-except LookupError:
-    nltk.download('punkt')
+import os
 
 # Initialize pyphen for syllable splitting
 dic = pyphen.Pyphen(lang='en')
@@ -200,9 +193,36 @@ class PronunciationAnalyzer:
     
     def tokenize_text(self, text: str) -> List[str]:
         """Tokenize text into words while preserving punctuation"""
-        # Use NLTK for better tokenization
-        tokens = word_tokenize(text)
-        return tokens
+        # Enhanced regex-based tokenization
+        # This pattern matches:
+        # - Words (including contractions like "don't")
+        # - Numbers
+        # - Punctuation marks
+        # - Preserves spacing
+        
+        # Split text into tokens while preserving structure
+        pattern = r"(?:\w+(?:'\w+)?|\d+|[^\w\s])"
+        tokens = re.findall(pattern, text)
+        
+        # Add spaces back where needed
+        result = []
+        text_pos = 0
+        
+        for token in tokens:
+            # Find the token's position in the original text
+            token_pos = text.find(token, text_pos)
+            
+            # Add any whitespace before the token
+            if token_pos > text_pos:
+                whitespace = text[text_pos:token_pos]
+                if whitespace.strip() == '':  # Only add if it's pure whitespace
+                    result.extend(list(whitespace))
+            
+            result.append(token)
+            text_pos = token_pos + len(token)
+        
+        # Filter out empty strings and normalize
+        return [token for token in result if token.strip()]
     
     def analyze_text(self, text: str) -> Tuple[List[Tuple[str, List[str]]], dict]:
         """Analyze text and return word-syllable pairs and statistics"""