Spaces:
Running
Running
File size: 1,073 Bytes
5ac8480 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | from camel_tools.tokenizers.word import simple_word_tokenize
from camel_tools.utils.dediac import dediac_ar
from camel_tools.utils.normalize import normalize_unicode
from camel_tools.utils.charmap import CharMapper
def clean_line(line, arclean):
return simple_word_tokenize(arclean(dediac_ar(normalize_unicode(line.strip()))))
def split_lines_words(lines):
return [line.strip().split() for line in lines]
def clean_mad(lines):
new_lines = []
for line in lines:
new_line = []
for word in line:
# replacing underscore with mad character
if word == 'ـ':
new_line.append('_')
else:
new_line.append(word)
new_lines.append(new_line)
return new_lines
def clean_lines(lines, arclean):
return [clean_line(line, arclean) for line in lines]
if __name__ == '__main__':
lines = []
with open('data/sample_text.txt', 'r') as f:
lines = f.readlines()
arclean = CharMapper.builtin_mapper("arclean")
new_lines = clean_lines(lines, arclean)
|