File size: 536 Bytes
e820a8a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | # utils.py
import re
import logging
def clean_text(text):
# Remove special characters and multiple spaces
text = re.sub(r'[^\w\s\u0600-\u06FF]', ' ', text)
text = re.sub(r'\s+', ' ', text).strip()
return text
def setup_logger(name):
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
|