from __future__ import annotations import json import logging import re from pathlib import Path from typing import Any logger = logging.getLogger(__name__) PROJECT_ROOT = Path(__file__).resolve().parents[2] LEXICON_ROOT = PROJECT_ROOT / "resources" / "lexicons" def lexicon_path(*parts: str) -> Path: """Kembalikan path absolut ke file/folder di resources/lexicons.""" return LEXICON_ROOT.joinpath(*parts) def _iter_clean_lines(path: Path) -> list[str]: if not path.exists(): logger.debug("Lexicon tidak ditemukan: %s", path) return [] lines: list[str] = [] try: with path.open(encoding="utf-8", errors="replace") as f: for raw in f: line = raw.strip().lstrip("\ufeff") if not line or line.startswith("#"): continue lines.append(line) except Exception as exc: logger.warning("Gagal membaca lexicon %s: %s", path, exc) return lines def load_word_set(*parts: str, lower: bool = True) -> set[str]: """Muat file newline-separated menjadi set kata/frasa.""" words: set[str] = set() for line in _iter_clean_lines(lexicon_path(*parts)): word = line.strip() if word: words.add(word.lower() if lower else word) return words def load_mapping( *parts: str, delimiter: str = "\t", lower_keys: bool = True, lower_values: bool = False, ) -> dict[str, str]: """Muat file mapping dua kolom, default TSV: keyvalue.""" mapping: dict[str, str] = {} for line in _iter_clean_lines(lexicon_path(*parts)): if delimiter not in line: logger.debug("Baris mapping dilewati (%s): %s", delimiter, line) continue key, value = line.split(delimiter, 1) key = key.strip() value = value.strip() if not key or not value: continue if lower_keys: key = key.lower() if lower_values: value = value.lower() if key != value.lower(): mapping[key] = value return mapping def load_json(*parts: str, default: Any = None) -> Any: """Muat JSON dari resources/lexicons dengan default bila gagal.""" path = lexicon_path(*parts) if not path.exists(): return default try: return json.loads(path.read_text(encoding="utf-8")) except Exception as exc: logger.warning("Gagal membaca JSON lexicon %s: %s", path, exc) return default def regex_alternation(words: set[str], *, phrase_spaces: bool = True) -> str: """ Buat alternation regex dari daftar kata/frasa. Frasa disortir dari paling panjang agar regex memilih match paling spesifik. """ parts: list[str] = [] for word in sorted((w for w in words if w), key=lambda w: (-len(w), w.lower())): escaped = re.escape(word) if phrase_spaces: escaped = re.sub(r"\\\s+", r"\\s+", escaped) parts.append(escaped) return "|".join(parts)