sacar tokenizer de ntlk

app.py

@@ -2,9 +2,7 @@ from gensim.models.keyedvectors import KeyedVectors
 import pickle
 from nltk.tokenize import word_tokenize
 import gradio as gr
-import nltk
 
-nltk.download('punkt')
 
 # Use gensim Keyvectors to read the embbedings 
 wordvectors_file_vec = 'smaller_model_spa.txt'
@@ -14,7 +12,7 @@ with open('stop_words.pkl', 'rb') as f:
 
 
 def filter_words(x):
-    word_tokens = word_tokenize(x, language = "spanish")
+    word_tokens = x.split(' ')  # shitty tokenization because ntlk tokenize on hf is working weeeird
     filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]
     return filtered_sentence