""" Handcrafted linguistic features for hybrid LCP models. Literature: word frequency is the strongest single predictor; WordNet senses capture ambiguity; syntax score captures structural difficulty. """ from __future__ import annotations import math from functools import lru_cache import numpy as np import wordfreq from syntax_complexity import analyze_syntax FEATURE_NAMES = [ "log_word_frequency", "word_length", "syllable_estimate", "wordnet_senses", "syntax_complexity", ] N_LINGUISTIC_FEATURES = len(FEATURE_NAMES) @lru_cache(maxsize=1) def _wordnet_ready() -> bool: try: from nltk.corpus import wordnet as wn wn.synsets("test") return True except LookupError: import nltk nltk.download("wordnet", quiet=True) nltk.download("omw-1.4", quiet=True) return True def _syllable_estimate(word: str) -> float: word = word.lower() if not word: return 0.0 vowels = "aeiouy" count = 0 prev_vowel = False for ch in word: is_vowel = ch in vowels if is_vowel and not prev_vowel: count += 1 prev_vowel = is_vowel return float(max(1, count)) def _wordnet_sense_count(word: str) -> float: _wordnet_ready() from nltk.corpus import wordnet as wn return float(len(wn.synsets(word.lower()))) def extract_linguistic_features(sentence: str, target_word: str) -> np.ndarray: """Return a 5-dim feature vector for one (sentence, target_word) pair.""" word = str(target_word).strip() freq = wordfreq.word_frequency(word.lower(), "en") log_freq = math.log10(freq + 1e-12) word_len = len(word) syllables = _syllable_estimate(word) senses = _wordnet_sense_count(word) syntax = analyze_syntax(str(sentence)).complexity_score return np.array([log_freq, word_len, syllables, senses, syntax], dtype=np.float32) def normalize_features(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """Z-score normalize feature matrix; return normalized, mean, std.""" mean = matrix.mean(axis=0) std = matrix.std(axis=0) std = np.where(std < 1e-6, 1.0, std) return (matrix - mean) / std, mean, std