cert_study_app/services/text_cleanup_service.py

import re


NOISE_ONLY_RE = re.compile(r"^[ㅇ○●◯θ∅µㆁ①②③④⑤⑥⑦⑧⑨⓵⓶⓷⓸⓹⓺⓻⓼⓽OΔXx]+$")
PAGE_FOOTER_RE = re.compile(r"^\d{1,3}\s*(?:[I|l]|\||/)\s*\d{1,3}$")


def is_noise_line(line: str) -> bool:
    text = re.sub(r"\s+", "", str(line or "").strip())
    if not text:
        return True
    if NOISE_ONLY_RE.fullmatch(text):
        return True
    if PAGE_FOOTER_RE.fullmatch(str(line or "").strip()):
        return True
    return False


def clean_question_text(value: str) -> str:
    if not value:
        return ""

    cleaned = []
    for raw_line in str(value).splitlines():
        line = re.sub(r"[ \u00A0\t]+", " ", raw_line).strip()
        if is_noise_line(line):
            continue
        cleaned.append(line)

    text = "\n".join(cleaned).strip()
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text


def clean_inline_text(value: str) -> str:
    text = clean_question_text(value)
    text = re.sub(r"\s+", " ", text).strip()
    return text