amank commited on
Commit ·
7007f93
1
Parent(s): 7839b8e
Minor Code cleanup in utils file
Browse files
utils.py
CHANGED
|
@@ -42,15 +42,8 @@ def keep_devnagri_hf_doc(document):
|
|
| 42 |
return cleaned_text
|
| 43 |
|
| 44 |
if batched:
|
| 45 |
-
|
| 46 |
-
cleaned_text_ls = []
|
| 47 |
-
for text in text_ls:
|
| 48 |
-
cleaned_text = get_clean_text(text)
|
| 49 |
-
cleaned_text_ls.append(cleaned_text)
|
| 50 |
-
document['text'] = cleaned_text_ls
|
| 51 |
else:
|
| 52 |
-
text = document['text']
|
| 53 |
-
cleaned_text = get_clean_text(text)
|
| 54 |
-
document['text'] = cleaned_text
|
| 55 |
|
| 56 |
return document
|
|
|
|
| 42 |
return cleaned_text
|
| 43 |
|
| 44 |
if batched:
|
| 45 |
+
document['text'] = [get_clean_text(text) for text in document['text']]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
else:
|
| 47 |
+
document['text'] = get_clean_text(document['text'])
|
|
|
|
|
|
|
| 48 |
|
| 49 |
return document
|