plagdetect/normalize.py

"""LAYER 1 of the AI/plagiarism defence: de-obfuscation & normalization.

Counters evasion tricks T4 (unicode homoglyphs) and T9 (invisible characters,
whitespace games) BEFORE any detector runs. Crucially, finding these tricks is
itself forensic evidence: nobody types Cyrillic 'a' inside an English essay by
accident. Returns the cleaned text plus an obfuscation-evidence report.
"""
import re
import unicodedata

# Characters that are invisible or zero-width: their only realistic purpose in
# a submitted document is breaking exact-match detectors.
ZERO_WIDTH = {
    "​": "ZERO WIDTH SPACE",
    "‌": "ZERO WIDTH NON-JOINER",
    "‍": "ZERO WIDTH JOINER",
    "⁠": "WORD JOINER",
    "": "ZERO WIDTH NO-BREAK SPACE (BOM)",
    "­": "SOFT HYPHEN",
    "͏": "COMBINING GRAPHEME JOINER",
    "᠎": "MONGOLIAN VOWEL SEPARATOR",
    "؜": "ARABIC LETTER MARK",
    "‎": "LEFT-TO-RIGHT MARK",
    "‏": "RIGHT-TO-LEFT MARK",
    "‪": "LEFT-TO-RIGHT EMBEDDING",
    "‫": "RIGHT-TO-LEFT EMBEDDING",
    "‬": "POP DIRECTIONAL FORMATTING",
    "‭": "LEFT-TO-RIGHT OVERRIDE",
    "‮": "RIGHT-TO-LEFT OVERRIDE",
    "⁡": "FUNCTION APPLICATION",
    "⁢": "INVISIBLE TIMES",
    "⁣": "INVISIBLE SEPARATOR",
    "⁤": "INVISIBLE PLUS",
}

# Cyrillic/Greek letters that render identically (or near identically) to
# Latin — the classic homoglyph substitution attack.
HOMOGLYPHS = {
    # Cyrillic lowercase
    "а": "a", "е": "e", "о": "o", "р": "p",
    "с": "c", "у": "y", "х": "x", "і": "i",
    "ј": "j", "ѕ": "s", "ԛ": "q", "ԝ": "w",
    "ё": "e", "ї": "i",
    # Cyrillic uppercase
    "А": "A", "В": "B", "Е": "E", "К": "K",
    "М": "M", "Н": "H", "О": "O", "Р": "P",
    "С": "C", "Т": "T", "У": "Y", "Х": "X",
    "Ѕ": "S", "І": "I", "Ј": "J", "Ԛ": "Q",
    "Ԝ": "W",
    # Greek (visually identical/near-identical pairs only)
    "Α": "A", "Β": "B", "Ε": "E", "Ζ": "Z",
    "Η": "H", "Ι": "I", "Κ": "K", "Μ": "M",
    "Ν": "N", "Ο": "O", "Ρ": "P", "Τ": "T",
    "Υ": "Y", "Χ": "X",
    "ο": "o", "ι": "i", "κ": "k", "ν": "v",
    "ρ": "p", "α": "a", "τ": "t", "υ": "u",
    # other lookalikes
    "ı": "i",   # dotless i
}

SPACE_VARIANTS = re.compile("[  -   　]")
QUOTE_SINGLE = re.compile("[‘’‚‛′]")
QUOTE_DOUBLE = re.compile("[“”„‟″]")
HYPHEN_VARIANTS = re.compile("[‐‑‒−]")

_LATIN_CONTEXT = re.compile(r"[A-Za-z]")


def deobfuscate(text):
    """Normalize text and report every evasion artifact found.

    Returns (clean_text, report) where report carries counts + examples so the
    UI can show "obfuscation evidence" (proof of an evasion attempt).
    """
    report = {
        "zero_width_count": 0,
        "zero_width_kinds": {},
        "homoglyph_count": 0,
        "homoglyph_examples": [],
        "invisible_per_10k_chars": 0.0,
        "spoof_suspected": False,
    }
    if not text:
        return text, report
    n_orig = len(text)

    # 1. strip zero-width / invisible characters, counting each kind
    kinds = {}
    found = 0
    for ch, name in ZERO_WIDTH.items():
        c = text.count(ch)
        if c:
            kinds[name] = c
            found += c
            text = text.replace(ch, "")
    report["zero_width_count"] = found
    report["zero_width_kinds"] = kinds

    # 2. homoglyph folding — only meaningful in Latin-script documents, so
    #    require the document to be predominantly Latin before folding
    latin_ratio = len(_LATIN_CONTEXT.findall(text[:4000])) / max(1, len(text[:4000]))
    if latin_ratio > 0.30:
        out = []
        examples = []
        homo = 0
        for i, ch in enumerate(text):
            rep = HOMOGLYPHS.get(ch)
            if rep is not None:
                homo += 1
                if len(examples) < 12:
                    ctx = text[max(0, i - 18):i + 18].replace("\n", " ")
                    examples.append(
                        {"char": ch, "codepoint": f"U+{ord(ch):04X}",
                         "name": unicodedata.name(ch, "?"),
                         "folded_to": rep, "context": ctx})
                out.append(rep)
            else:
                out.append(ch)
        text = "".join(out)
        report["homoglyph_count"] = homo
        report["homoglyph_examples"] = examples

    # 3. compatibility normalization (ligatures fi->fi, fullwidth, etc.)
    text = unicodedata.normalize("NFKC", text)

    # 4. cosmetic normalization (keeps em/en dashes — they are a style signal)
    text = SPACE_VARIANTS.sub(" ", text)
    text = QUOTE_SINGLE.sub("'", text)
    text = QUOTE_DOUBLE.sub('"', text)
    text = HYPHEN_VARIANTS.sub("-", text)

    invisible_density = 10_000.0 * report["zero_width_count"] / max(1, n_orig)
    report["invisible_per_10k_chars"] = round(invisible_density, 2)
    # thresholds: a stray BOM or one soft hyphen is normal from PDF export;
    # repeated zero-width chars or ANY homoglyph in Latin text is deliberate
    report["spoof_suspected"] = bool(
        report["homoglyph_count"] >= 2
        or report["zero_width_count"] >= 8
        or invisible_density >= 5.0)
    return text, report