plagdetect/textutils.py

"""Text utilities: tokenization, hashing vectorizer, synonym transforms."""
import re
import zlib

import numpy as np

STOPWORDS = set("""a an the of in on at to for from by with and or is are was were be been
this that these those it its as we our their his her they i you not no than then so such
can could may might will would should has have had do does did into over under more most
other which who whom what when where how also between among using used use both each per
""".split())

HEDGE_WORDS = {"may", "might", "suggests", "suggest", "preliminary", "possibly",
               "appears", "likely", "indicate", "indicates", "potentially"}
STRONG_WORDS = {"demonstrates", "proves", "clearly", "significantly", "definitively",
                "guarantees", "always", "ensures", "establishes", "confirms"}

SYNONYMS = {
    "approach": "methodology", "demonstrates": "shows", "significant": "substantial",
    "framework": "architecture", "evaluate": "assess", "propose": "introduce",
    "results": "findings", "improve": "enhance", "secure": "protect",
    "data": "information", "method": "technique", "novel": "new",
    "utilize": "employ", "performance": "effectiveness", "robust": "resilient",
    "analysis": "examination", "system": "platform", "challenge": "obstacle",
    "crucial": "critical", "rapid": "fast", "growth": "expansion",
    "reduce": "decrease", "increase": "raise", "achieve": "attain",
    "ensure": "guarantee", "existing": "current", "various": "diverse",
}


def tokenize(text):
    """Lowercase word tokens."""
    return re.findall(r"[a-z0-9][a-z0-9\-']*", text.lower())


def content_tokens(text):
    return [t for t in tokenize(text) if t not in STOPWORDS and len(t) > 2]


def sentences(text):
    """Split text into sentences (robust to abbreviations like e.g., et al.)."""
    text = re.sub(r"\s+", " ", text).strip()
    parts = re.split(r"(?<![A-Z])(?<!al)(?<!e\.g)(?<!i\.e)\.\s+(?=[A-Z])", text)
    return [p.strip().rstrip(".") + "." for p in parts if len(p.strip()) > 10]


def crc32h(s):
    return zlib.crc32(s.encode("utf-8")) & 0xFFFFFFFF


def hash_vec(text, dim=2048):
    """Hashed character-trigram count vector (stable across processes)."""
    dim = int(dim)  # crc32h() can exceed 2^31; a numpy int32 dim (from a loaded
    # model's sizes) would overflow "% dim" on Windows (C long is 32-bit).
    v = np.zeros(dim, dtype=np.float64)
    t = " " + re.sub(r"\s+", " ", text.lower()) + " "
    for i in range(len(t) - 2):
        v[crc32h(t[i:i + 3]) % dim] += 1.0
    n = np.linalg.norm(v)
    return v / n if n > 0 else v


def hash_matrix(texts, dim=2048):
    return np.stack([hash_vec(t, dim) for t in texts]) if texts else np.zeros((0, dim))


def squash(x, scale=1.0):
    """Map [0, inf) -> [0, 1) smoothly."""
    x = max(0.0, float(x)) * scale
    return x / (1.0 + x)


def jaccard(a, b):
    a, b = set(a), set(b)
    if not a and not b:
        return 0.0
    return len(a & b) / max(1, len(a | b))


def synonymize(text, rng, p=0.85):
    """Find-replace transform: substitute known synonyms with probability p."""
    out = []
    for w in re.split(r"(\W+)", text):
        lw = w.lower()
        if lw in SYNONYMS and rng.rand() < p:
            rep = SYNONYMS[lw]
            out.append(rep.capitalize() if w[:1].isupper() else rep)
        else:
            out.append(w)
    return "".join(out)


def mosaic_mix(src_sents, other_sents, rng, frac_src=0.55):
    """Mosaic transform: weave source sentences with sentences from elsewhere."""
    n = max(4, len(src_sents))
    take_src = max(2, int(round(n * frac_src)))
    picked = [s for s in rng.permutation(src_sents)[:take_src]]
    fill = [s for s in rng.permutation(other_sents)[:n - take_src]]
    mixed = picked + fill
    rng.shuffle(mixed)
    return " ".join(synonymize(s, rng, p=0.3) for s in mixed)