Download src/root_validator.rs from veripazari/durak: direct link, hf CLI and curl.
- Browser
- Download file 7.23 kB
-
https://huggingface.co/veripazari/durak/resolve/main/src/root_validator.rs
- Command line
-
hf download hf://veripazari/durak/src/root_validator.rs
-
curl -L -o root_validator.rs https://huggingface.co/veripazari/durak/resolve/main/src/root_validator.rs
7.23 kB
| //! Root Validity Checker for Turkish morphological analysis | |
| //! | |
| //! Validates candidate roots after suffix stripping to prevent | |
| //! over-stripping and ensure linguistically valid outputs. | |
| use std::collections::HashSet; | |
| use std::sync::OnceLock; | |
| /// Turkish vowels (both lowercase and uppercase) | |
| const TURKISH_VOWELS: &[char] = &[ | |
| 'a', 'e', 'ı', 'i', 'o', 'ö', 'u', 'ü', 'A', 'E', 'I', 'İ', 'O', 'Ö', 'U', 'Ü', | |
| ]; | |
| /// Turkish sonorant consonants (can end words naturally) | |
| const SONORANT_CONSONANTS: &[char] = &['l', 'r', 'n', 'm', 'y', 'L', 'R', 'N', 'M', 'Y']; | |
| /// Turkish voiceless stops (words typically don't end in these without specific patterns) | |
| const VOICELESS_STOPS: &[char] = &['p', 'ç', 't', 'k', 'P', 'Ç', 'T', 'K']; | |
| /// Impossible Turkish consonant clusters at word end | |
| const INVALID_FINAL_CLUSTERS: &[&str] = &[ | |
| "çk", "çp", "çt", "ğk", "ğp", "ğt", "kb", "kc", "kç", "kg", "kğ", "kj", "pb", "pc", "pç", "pg", | |
| "pğ", "pj", "tb", "tc", "tç", "tg", "tğ", "tj", "nd", "nt", "nk", "ng", | |
| ]; | |
| /// Common Turkish bound stems - these look like valid roots but aren't free words | |
| const BOUND_STEMS: &[&str] = &[ | |
| "öğrenc", "öğret", "tanı", "yapı", "bili", "geli", "gidi", "kali", "veri", "alı", "olı", | |
| "görü", "bilü", | |
| ]; | |
| /// Embedded valid roots from lemma dictionary | |
| static LEMMA_DICT_DATA: &str = include_str!("../resources/tr/lemmas/turkish_lemma_dict.txt"); | |
| static VALID_ROOTS: OnceLock<HashSet<String>> = OnceLock::new(); | |
| /// Get valid root words from lemma dictionary | |
| pub fn get_valid_roots() -> &'static HashSet<String> { | |
| VALID_ROOTS.get_or_init(|| { | |
| let mut roots = HashSet::new(); | |
| for line in LEMMA_DICT_DATA.lines() { | |
| let line = line.trim(); | |
| if line.is_empty() || line.starts_with('#') { | |
| continue; | |
| } | |
| if let Some((_, lemma)) = line.split_once('\t') { | |
| roots.insert(lemma.trim().to_string()); | |
| } | |
| } | |
| roots | |
| }) | |
| } | |
| /// Root validity checker for Turkish morphology | |
| pub struct RootValidator { | |
| /// Minimum acceptable root length (characters) | |
| pub min_root_length: usize, | |
| /// Whether to use strict dictionary checking | |
| pub strict: bool, | |
| } | |
| impl Default for RootValidator { | |
| fn default() -> Self { | |
| Self { | |
| min_root_length: 2, | |
| strict: false, | |
| } | |
| } | |
| } | |
| impl RootValidator { | |
| /// Create a new validator with custom settings | |
| pub fn new(min_root_length: usize, strict: bool) -> Self { | |
| Self { | |
| min_root_length, | |
| strict, | |
| } | |
| } | |
| /// Check if a candidate root is valid | |
| pub fn is_valid_root(&self, candidate: &str) -> bool { | |
| // 1. Minimum length check | |
| if candidate.chars().count() < self.min_root_length { | |
| return false; | |
| } | |
| // 2. Check if it's a known bound stem (never valid) | |
| let candidate_lower = candidate.to_lowercase(); | |
| for bound_stem in BOUND_STEMS { | |
| if candidate_lower == *bound_stem || candidate_lower.ends_with(bound_stem) { | |
| return false; | |
| } | |
| } | |
| // 3. Strict mode: must be in known roots dictionary | |
| if self.strict { | |
| return get_valid_roots().contains(candidate); | |
| } | |
| // 4. Lenient mode: check linguistic constraints | |
| self.check_phonotactics(candidate) | |
| } | |
| /// Check Turkish phonotactic constraints | |
| fn check_phonotactics(&self, word: &str) -> bool { | |
| if word.is_empty() { | |
| return false; | |
| } | |
| let chars: Vec<char> = word.chars().collect(); | |
| let word_lower: String = word.to_lowercase(); | |
| let lower_chars: Vec<char> = word_lower.chars().collect(); | |
| // Must contain at least one vowel | |
| if !chars.iter().any(|c| TURKISH_VOWELS.contains(c)) { | |
| return false; | |
| } | |
| // Check for invalid final consonant clusters | |
| for cluster in INVALID_FINAL_CLUSTERS { | |
| if word_lower.ends_with(cluster) { | |
| return false; | |
| } | |
| } | |
| // Check if last char is a vowel - always valid | |
| let last_char = lower_chars.last().unwrap(); | |
| if TURKISH_VOWELS.contains(last_char) { | |
| return true; | |
| } | |
| // Check if last char is a sonorant - usually valid | |
| if SONORANT_CONSONANTS.contains(last_char) { | |
| return self.has_valid_syllable_structure(&lower_chars); | |
| } | |
| // Voiceless stops at word end are valid in Turkish, including | |
| // monosyllabic CVC roots such as "at", "it", "ot", "et". | |
| if VOICELESS_STOPS.contains(last_char) { | |
| return lower_chars.len() >= 2 && self.has_valid_syllable_structure(&lower_chars); | |
| } | |
| true | |
| } | |
| /// Check if the word has a valid Turkish syllable structure | |
| fn has_valid_syllable_structure(&self, chars: &[char]) -> bool { | |
| if chars.len() < 2 { | |
| return false; | |
| } | |
| let vowel_count = chars.iter().filter(|c| TURKISH_VOWELS.contains(c)).count(); | |
| if vowel_count == 0 { | |
| return false; | |
| } | |
| if chars.len() <= 3 { | |
| return vowel_count >= 1; | |
| } | |
| let vowel_ratio = vowel_count as f32 / chars.len() as f32; | |
| vowel_ratio >= 0.2 && vowel_ratio <= 0.7 | |
| } | |
| } | |
| mod tests { | |
| use super::*; | |
| fn test_min_length() { | |
| let validator = RootValidator::new(2, false); | |
| assert!(!validator.is_valid_root("k")); | |
| assert!(validator.is_valid_root("ki")); | |
| assert!(validator.is_valid_root("kitap")); | |
| } | |
| fn test_vowel_requirement() { | |
| let validator = RootValidator::default(); | |
| assert!(!validator.is_valid_root("krm")); | |
| assert!(validator.is_valid_root("karma")); | |
| } | |
| fn test_invalid_clusters() { | |
| let validator = RootValidator::default(); | |
| assert!(!validator.is_valid_root("kitaçk")); | |
| assert!(validator.is_valid_root("kitap")); | |
| } | |
| fn test_bound_stems_rejected() { | |
| let validator = RootValidator::default(); | |
| assert!(!validator.is_valid_root("öğrenc")); | |
| assert!(!validator.is_valid_root("ÖĞRENC")); | |
| } | |
| fn test_valid_word_endings() { | |
| let validator = RootValidator::default(); | |
| assert!(validator.is_valid_root("kita")); | |
| assert!(validator.is_valid_root("evi")); | |
| assert!(validator.is_valid_root("kal")); | |
| assert!(validator.is_valid_root("gel")); | |
| assert!(validator.is_valid_root("gün")); | |
| assert!(validator.is_valid_root("kitap")); | |
| assert!(validator.is_valid_root("at")); | |
| } | |
| fn test_strict_mode() { | |
| let validator = RootValidator::new(2, true); | |
| assert!(validator.is_valid_root("kitap")); | |
| assert!(validator.is_valid_root("ev")); | |
| assert!(!validator.is_valid_root("xyzabc")); | |
| } | |
| fn test_known_roots() { | |
| let roots = get_valid_roots(); | |
| assert!(roots.contains("kitap")); | |
| assert!(roots.contains("ev")); | |
| assert!(roots.contains("gel")); | |
| assert!(roots.contains("git")); | |
| } | |
| } | |