Sources/WtpsplitKit/CharClasses.swift

import Foundation

/// Character-class helpers shared by tokenization and the pause-aware mask.
///
/// All text operations in this kit work on **Unicode scalars** (code points),
/// not grapheme clusters, so that lengths and split indices match Python's
/// `len(str)` / `str[i:j]` semantics exactly (the reference implementation in
/// `scripts/run_segmentation.py`).
enum CharClasses {

    /// CJK ranges used by the reference implementation (`run_segmentation.py`).
    /// CJK Unified, Extension A, Compatibility Ideographs, CJK symbols/punct,
    /// and the half/full-width forms block.
    private static let cjkRanges: [ClosedRange<UInt32>] = [
        0x4E00...0x9FFF,
        0x3400...0x4DBF,
        0xF900...0xFAFF,
        0x3000...0x303F,
        0xFF00...0xFFEF,
    ]

    @inline(__always)
    static func isCJK(_ s: Unicode.Scalar) -> Bool {
        let cp = s.value
        for r in cjkRanges where r.contains(cp) { return true }
        return false
    }

    /// Mirrors Python's `str.isspace()` closely enough for segmentation: Unicode
    /// whitespace plus the ASCII control separators Python treats as space.
    @inline(__always)
    static func isSpace(_ s: Unicode.Scalar) -> Bool {
        if s.properties.isWhitespace { return true }
        switch s.value {
        case 0x09...0x0D, 0x1C...0x1F, 0x85: return true
        default: return false
        }
    }

    /// Python `string.punctuation`: !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
    static let asciiPunctuation: Set<Unicode.Scalar> = {
        let chars = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
        return Set(chars.unicodeScalars)
    }()

    /// Clause-level pause punctuation (comma / semicolon / colon / brackets /
    /// dashes, plus their CJK forms and closing quotes). From `CLAUSE_PUNCT`.
    static let clausePunct: Set<Unicode.Scalar> = {
        let chars = ",;:)]}—–"            // , ; : ) ] } em/en dash
            + "，、；："                    // CJK , 、 ; :
            + "”’"                          // closing curly quotes
        return Set(chars.unicodeScalars)
    }()

    /// CJK sentence-ending punctuation. From `CJK_SENT_PUNCT`.
    static let cjkSentencePunct: Set<Unicode.Scalar> = {
        Set("。！？…".unicodeScalars)
    }()

    /// Clause/phrase-introducing words; breaking *before* one reads more
    /// naturally than a bare word gap. From `CONNECTORS`.
    static let connectors: Set<String> = [
        "and", "but", "or", "nor", "yet", "so", "for",
        "which", "that", "who", "whom", "whose", "where", "when", "while",
        "because", "although", "though", "since", "if", "unless", "until",
        "after", "before", "as", "than", "whether",
    ]
}