core/constants.py

"""
Configuration constants and hyperparameters for the SinCode engine.
"""

import re

# ─── Model & Data Paths ─────────────────────────────────────────────────────

DEFAULT_MODEL_NAME = "FacebookAI/xlm-roberta-base"
DEFAULT_DICTIONARY_PATH = "dictionary.pkl"

ENGLISH_CORPUS_URL = (
    "https://raw.githubusercontent.com/first20hours/google-10000-english/master/20k.txt"
)

# ─── Scoring Weights (tunable hyperparameters) ──────────────────────────────

W_MLM: float = 0.55           # Contextual language model probability
W_FIDELITY: float = 0.45      # Source-aware transliteration fidelity
W_RANK: float = 0.00          # Dictionary rank prior (disabled — dict is unordered)

# ─── Decoding Parameters ────────────────────────────────────────────────────

MAX_CANDIDATES: int = 8       # Max candidates per word position
DEFAULT_BEAM_WIDTH: int = 5   # Beam search width
FIDELITY_SCALE: float = 10.0  # Edit-distance penalty multiplier
DICT_FIDELITY_DAMP: float = 2.0  # Decay rate for dict bonus (higher = stricter filter)
MIN_ENGLISH_LEN: int = 3      # Min word length for 20k-corpus English detection

# ─── Unicode Constants ──────────────────────────────────────────────────────

SINHALA_VIRAMA: str = '\u0DCA'  # Sinhala virama (hal) character
ZWJ: str = '\u200D'             # Zero-width joiner (for conjuncts)

# ─── Regex ──────────────────────────────────────────────────────────────────

PUNCT_PATTERN = re.compile(r"^(\W*)(.*?)(\W*)$")