src/core/lexicons.py

from __future__ import annotations
import json
import logging
import re
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
PROJECT_ROOT = Path(__file__).resolve().parents[2]
LEXICON_ROOT = PROJECT_ROOT / "resources" / "lexicons"

def lexicon_path(*parts: str) -> Path:
    """Kembalikan path absolut ke file/folder di resources/lexicons."""
    return LEXICON_ROOT.joinpath(*parts)

def _iter_clean_lines(path: Path) -> list[str]:
    if not path.exists():
        logger.debug("Lexicon tidak ditemukan: %s", path)
        return []
    lines: list[str] = []
    try:
        with path.open(encoding="utf-8", errors="replace") as f:
            for raw in f:
                line = raw.strip().lstrip("\ufeff")
                if not line or line.startswith("#"):
                    continue
                lines.append(line)
    except Exception as exc:
        logger.warning("Gagal membaca lexicon %s: %s", path, exc)
    return lines

def load_word_set(*parts: str, lower: bool = True) -> set[str]:
    """Muat file newline-separated menjadi set kata/frasa."""
    words: set[str] = set()
    for line in _iter_clean_lines(lexicon_path(*parts)):
        word = line.strip()
        if word:
            words.add(word.lower() if lower else word)
    return words

def load_mapping(
    *parts: str,
    delimiter: str = "\t",
    lower_keys: bool = True,
    lower_values: bool = False,
) -> dict[str, str]:
    """Muat file mapping dua kolom, default TSV: key<TAB>value."""
    mapping: dict[str, str] = {}
    for line in _iter_clean_lines(lexicon_path(*parts)):
        if delimiter not in line:
            logger.debug("Baris mapping dilewati (%s): %s", delimiter, line)
            continue
        key, value = line.split(delimiter, 1)
        key = key.strip()
        value = value.strip()
        if not key or not value:
            continue
        if lower_keys:
            key = key.lower()
        if lower_values:
            value = value.lower()
        if key != value.lower():
            mapping[key] = value
    return mapping

def load_json(*parts: str, default: Any = None) -> Any:
    """Muat JSON dari resources/lexicons dengan default bila gagal."""
    path = lexicon_path(*parts)
    if not path.exists():
        return default
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except Exception as exc:
        logger.warning("Gagal membaca JSON lexicon %s: %s", path, exc)
        return default

def regex_alternation(words: set[str], *, phrase_spaces: bool = True) -> str:
    """
    Buat alternation regex dari daftar kata/frasa.
    Frasa disortir dari paling panjang agar regex memilih match paling spesifik.
    """
    parts: list[str] = []
    for word in sorted((w for w in words if w), key=lambda w: (-len(w), w.lower())):
        escaped = re.escape(word)
        if phrase_spaces:
            escaped = re.sub(r"\\\s+", r"\\s+", escaped)
        parts.append(escaped)
    return "|".join(parts)