src/linguistic_features.py

"""
Handcrafted linguistic features for hybrid LCP models.

Literature: word frequency is the strongest single predictor; WordNet senses
capture ambiguity; syntax score captures structural difficulty.
"""

from __future__ import annotations

import math
from functools import lru_cache

import numpy as np
import wordfreq

from syntax_complexity import analyze_syntax

FEATURE_NAMES = [
    "log_word_frequency",
    "word_length",
    "syllable_estimate",
    "wordnet_senses",
    "syntax_complexity",
]

N_LINGUISTIC_FEATURES = len(FEATURE_NAMES)


@lru_cache(maxsize=1)
def _wordnet_ready() -> bool:
    try:
        from nltk.corpus import wordnet as wn

        wn.synsets("test")
        return True
    except LookupError:
        import nltk

        nltk.download("wordnet", quiet=True)
        nltk.download("omw-1.4", quiet=True)
        return True


def _syllable_estimate(word: str) -> float:
    word = word.lower()
    if not word:
        return 0.0
    vowels = "aeiouy"
    count = 0
    prev_vowel = False
    for ch in word:
        is_vowel = ch in vowels
        if is_vowel and not prev_vowel:
            count += 1
        prev_vowel = is_vowel
    return float(max(1, count))


def _wordnet_sense_count(word: str) -> float:
    _wordnet_ready()
    from nltk.corpus import wordnet as wn

    return float(len(wn.synsets(word.lower())))


def extract_linguistic_features(sentence: str, target_word: str) -> np.ndarray:
    """Return a 5-dim feature vector for one (sentence, target_word) pair."""
    word = str(target_word).strip()
    freq = wordfreq.word_frequency(word.lower(), "en")
    log_freq = math.log10(freq + 1e-12)
    word_len = len(word)
    syllables = _syllable_estimate(word)
    senses = _wordnet_sense_count(word)
    syntax = analyze_syntax(str(sentence)).complexity_score
    return np.array([log_freq, word_len, syllables, senses, syntax], dtype=np.float32)


def normalize_features(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Z-score normalize feature matrix; return normalized, mean, std."""
    mean = matrix.mean(axis=0)
    std = matrix.std(axis=0)
    std = np.where(std < 1e-6, 1.0, std)
    return (matrix - mean) / std, mean, std