Spaces:
Runtime error
Runtime error
removing stopwords from vocabulary scores
Browse files
app.py
CHANGED
|
@@ -428,6 +428,7 @@ def get_level(word):
|
|
| 428 |
word = word.strip(" ")
|
| 429 |
data = json.loads(f.read())
|
| 430 |
level = 0
|
|
|
|
| 431 |
for k, v in data.items():
|
| 432 |
if word in v:
|
| 433 |
level = k
|
|
@@ -437,10 +438,17 @@ def get_level(word):
|
|
| 437 |
|
| 438 |
def vocab_level_inter(text):
|
| 439 |
text = word_tokenize(text)
|
|
|
|
|
|
|
|
|
|
| 440 |
interp = [('',0)]
|
| 441 |
sum = 0
|
| 442 |
total = 0
|
| 443 |
for idx, i in enumerate(text):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 444 |
lvl = int(get_level(i))/4
|
| 445 |
interp.append((i, lvl))
|
| 446 |
sum+= lvl
|
|
|
|
| 428 |
word = word.strip(" ")
|
| 429 |
data = json.loads(f.read())
|
| 430 |
level = 0
|
| 431 |
+
|
| 432 |
for k, v in data.items():
|
| 433 |
if word in v:
|
| 434 |
level = k
|
|
|
|
| 438 |
|
| 439 |
def vocab_level_inter(text):
|
| 440 |
text = word_tokenize(text)
|
| 441 |
+
stop_words = set(stopwords.words('english'))
|
| 442 |
+
for i in string.punctuation:
|
| 443 |
+
stop_words.add(i)
|
| 444 |
interp = [('',0)]
|
| 445 |
sum = 0
|
| 446 |
total = 0
|
| 447 |
for idx, i in enumerate(text):
|
| 448 |
+
if i in stop_words:
|
| 449 |
+
lvl = -1
|
| 450 |
+
interp.append((i, lvl))
|
| 451 |
+
continue
|
| 452 |
lvl = int(get_level(i))/4
|
| 453 |
interp.append((i, lvl))
|
| 454 |
sum+= lvl
|