Update app.py

app.py

@@ -6,23 +6,27 @@ import pandas as pd
 import torch
 import nltk
 import time
+import subprocess
 from concurrent.futures import ThreadPoolExecutor
 
-
 from langchain_openai import ChatOpenAI
-
 from langchain.schema import SystemMessage, HumanMessage
 from sentence_transformers import SentenceTransformer, util
 
-# Load NLP libraries
+# Ensure necessary NLP models are available
+try:
+    nltk.data.find("tokenizers/punkt")
+except LookupError:
+    print("Downloading NLTK punkt tokenizer...")
+    nltk.download("punkt")
+
 try:
     import spacy
     nlp = spacy.load("en_core_web_sm")
-    use_spacy = True
-except Exception:
-    st.warning("SpaCy model not found, falling back to NLTK for tokenization.")
-    nltk.download("punkt")
-    use_spacy = False
+except OSError:
+    print("Downloading SpaCy model...")
+    subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
+    nlp = spacy.load("en_core_web_sm")
 
 # Load AI models
 translator = ChatOpenAI(model="gpt-3.5-turbo")
@@ -38,8 +42,8 @@ def load_glossary_from_excel(glossary_file_bytes) -> dict:
         if pd.notnull(row['English']) and pd.notnull(row['CanadianFrench']):
             english_term = row['English'].strip().lower()
             french_term = row['CanadianFrench'].strip()
-            doc = nlp(english_term) if use_spacy else english_term.split()
-            lemmatized_term = " ".join([token.lemma_ for token in doc]) if use_spacy else english_term
+            doc = nlp(english_term) if nlp else english_term.split()
+            lemmatized_term = " ".join([token.lemma_ for token in doc]) if nlp else english_term
             glossary[lemmatized_term] = french_term
 
     return dict(sorted(glossary.items(), key=lambda item: len(item[0]), reverse=True))
@@ -64,7 +68,7 @@ def retry_translate_text(text: str, max_retries=3) -> str:
             return response.content.strip()
         except Exception as e:
             print(f"Error in translation (attempt {attempt+1}): {e}")
-            time.sleep(2)  # Wait before retrying
+            time.sleep(2)
     return "Translation failed. Please try again later."
 
 def enforce_glossary(text: str, glossary: dict, threshold: float) -> str:
@@ -72,7 +76,7 @@ def enforce_glossary(text: str, glossary: dict, threshold: float) -> str:
     glossary_items = tuple(sorted(glossary.items()))
     glossary_terms, glossary_embeddings = compute_glossary_embeddings_cached(glossary_items)
 
-    sentences = nltk.tokenize.sent_tokenize(text) if not use_spacy else [sent.text for sent in nlp(text).sents]
+    sentences = nltk.tokenize.sent_tokenize(text) if not nlp else [sent.text for sent in nlp(text).sents]
     
     def process_sentence(sentence):
         """Processes a single sentence with glossary enforcement."""
@@ -95,7 +99,6 @@ def enforce_glossary(text: str, glossary: dict, threshold: float) -> str:
 
         return sentence.strip()
 
-    # Process sentences in parallel for speed
     with ThreadPoolExecutor() as executor:
         updated_sentences = list(executor.map(process_sentence, sentences))