File size: 2,547 Bytes
4211f06 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
import os
import re
from tqdm import tqdm
def clean_devanagari_text(text):
"""
Cleans the given Devanagari text by removing newline characters, extra spaces, and English letters.
Args:
text (str): The input text containing Devanagari script.
Returns:
str: The cleaned Devanagari text.
"""
# Remove newline characters
text = re.sub(r"\n", " ", text)
# Regex pattern to match anything that is not Devanagari letters, punctuation, or numbers
pattern = r'[^0-9\u0900-\u097F.,!?;:()\'"—-₹₹\s]'
text = re.sub(pattern, "", text)
# Regex pattern to match Devanagari words and separate it from other characters
pattern = r"([\u0900-\u097F]+)" # Matches Devanagari characters
text = re.sub(pattern, r" \1 ", text) # Add spaces around Devanagari words
# Add space before danda (।) and double danda (॥)
pattern = r"([\u0964\u0965])" # Matches danda and double danda
text = re.sub(pattern, r" \1 ", text) # Add spaces around danda and double danda
# Replace following : \u202c \u202a \u2061
text = re.sub(r"\u202c", "", text)
text = re.sub(r"\u202a", "", text)
text = re.sub(r"\u2061", "", text)
# Remove extra spaces
text = re.sub(r"\s+", " ", text).strip()
return text
def read_txt_files(folder_path="", clean_text=True, data_percentage_limit=1.0):
"""
Reads all .txt files in the given folder and returns their content as a list.
Args:
folder_path (str): The path to the folder containing .txt files.
Returns:
list: A list containing the content of each .txt file.
"""
content_list = []
num_files_to_read = int(len(os.listdir(folder_path)) * data_percentage_limit)
num_files_read = 0
# Iterate through all files in the folder
for filename in tqdm(os.listdir(folder_path)):
num_files_read += 1
if num_files_read > num_files_to_read:
break
else:
# Check if the file has a .txt extension
if filename.endswith(".txt"):
file_path = os.path.join(folder_path, filename)
# Open the file and read its content
with open(file_path, "r", encoding="utf-8") as file:
content = file.read()
if clean_text:
clean_content = clean_devanagari_text(content)
else:
clean_content = content
content_list.append(clean_content)
return content_list
|