"""Text normalization for de-duplication (spec §3.3.5)."""
from __future__ import annotations

import re

_LINE_COMMENT_RE = re.compile(r"//.*")
_BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL)
_WHITESPACE_RE = re.compile(r"\s+")


def strip_comments_and_whitespace(code: str) -> str:
    """Strips C/C++ comments then collapses all whitespace, for dedup hashing
    and shingling. Not a full lexer (doesn't special-case string/char literals
    containing `//` or `/*`) — acceptable for a dedup signal, not for
    correctness-sensitive code transforms."""
    code = _BLOCK_COMMENT_RE.sub(" ", code)
    code = _LINE_COMMENT_RE.sub(" ", code)
    code = _WHITESPACE_RE.sub(" ", code).strip()
    return code


def shingles(text: str, n: int = 5) -> set[str]:
    """Word n-grams ("5-gram") over the normalized text, for Jaccard/MinHash."""
    tokens = text.split(" ")
    if len(tokens) < n:
        return {" ".join(tokens)} if tokens else set()
    return {" ".join(tokens[i : i + n]) for i in range(len(tokens) - n + 1)}