|
|
"""
|
|
|
Moduł do ekstrakcji cech lingwistycznych i stylistycznych tekstu.
|
|
|
"""
|
|
|
import math
|
|
|
import re
|
|
|
from collections import Counter
|
|
|
from typing import Dict, List
|
|
|
|
|
|
from ..utils import safe_divide
|
|
|
from ..constants import STOP_WORDS, BAD_WORDS, NON_WORD_CHARS_PATTERN
|
|
|
|
|
|
|
|
|
|
|
|
def calculate_stop_word_ratio(words_lower: List[str]) -> Dict[str, float]:
|
|
|
"""Oblicza stosunek stop-words do wszystkich słów."""
|
|
|
total_words = len(words_lower)
|
|
|
stop_word_count = sum(1 for w in words_lower if w in STOP_WORDS)
|
|
|
return {'stop_word_ratio': safe_divide(stop_word_count, total_words)}
|
|
|
|
|
|
def count_bad_words(words_lower: List[str]) -> Dict[str, int]:
|
|
|
"""Liczy wystąpienia wulgaryzmów."""
|
|
|
return {'bad_word_count': sum(1 for w in words_lower if w in BAD_WORDS)}
|
|
|
|
|
|
def calculate_unigram_entropy(words_lower: List[str]) -> Dict[str, float]:
|
|
|
"""Oblicza entropię rozkładu unigramów (słów)."""
|
|
|
total_words = len(words_lower)
|
|
|
if not total_words: return {'entropy': 0.0}
|
|
|
counts = Counter(words_lower)
|
|
|
entropy = -sum((cnt / total_words) * math.log(cnt / total_words) for cnt in counts.values())
|
|
|
return {'entropy': entropy}
|
|
|
|
|
|
def count_non_alpha_words(text: str) -> Dict[str, float]:
|
|
|
"""Liczy stosunek znaków niealfabetycznych do wszystkich."""
|
|
|
total_chars = len(text)
|
|
|
if not total_chars: return {'non_alpha_word_fractions': 0.0}
|
|
|
non_alpha = sum(1 for char in text if not char.isalpha())
|
|
|
return {'non_alpha_word_fractions': safe_divide(non_alpha, total_chars)}
|
|
|
|
|
|
def calculate_symbol_to_word_ratio(words: List[str], text: str) -> Dict[str, float]:
|
|
|
"""Oblicza stosunek symboli do słów."""
|
|
|
total_words = len(words)
|
|
|
char_counts = Counter(text)
|
|
|
triple_dot_count = text.count('...')
|
|
|
symbol_count = char_counts.get('#', 0) + triple_dot_count + char_counts.get('…', 0)
|
|
|
return {'symbol_to_word_ratio': safe_divide(symbol_count, total_words)}
|
|
|
|
|
|
|
|
|
|
|
|
def calculate_ngram_fractions(words: List[str]) -> Dict[str, float]:
|
|
|
"""Analizuje frakcje powtarzających się i najczęstszych n-gramów."""
|
|
|
normalized_text = NON_WORD_CHARS_PATTERN.sub('', ' '.join(words))
|
|
|
total_chars = len(normalized_text)
|
|
|
|
|
|
keys = [f'fraction_duplicate_{n}_ngram' for n in range(5, 11)] + \
|
|
|
[f'fraction_top_{n}_ngram' for n in range(2, 6)]
|
|
|
if total_chars == 0 or len(words) < 10:
|
|
|
return {key: 0.0 for key in keys}
|
|
|
|
|
|
results = {}
|
|
|
for n in range(2, 11):
|
|
|
if len(words) < n: continue
|
|
|
ngrams_gen = (' '.join(words[i:i + n]) for i in range(len(words) - n + 1))
|
|
|
counts = Counter(ngrams_gen)
|
|
|
if n >= 5:
|
|
|
dup_chars = sum(len(ngram) for ngram, cnt in counts.items() if cnt > 1)
|
|
|
results[f'fraction_duplicate_{n}_ngram'] = safe_divide(dup_chars, total_chars)
|
|
|
if 2 <= n <= 5:
|
|
|
if counts:
|
|
|
top_ngram, _ = counts.most_common(1)[0]
|
|
|
results[f'fraction_top_{n}_ngram'] = safe_divide(len(top_ngram), total_chars)
|
|
|
else:
|
|
|
results[f'fraction_top_{n}_ngram'] = 0.0
|
|
|
return results
|
|
|
|
|
|
|
|
|
|
|
|
def analyze_stylistic_metrics(text: str, words: List[str], sentences: List[str]) -> Dict[str, float]:
|
|
|
sentences_from_regex = re.findall(r'[^.!?]+[.!?]', text)
|
|
|
num_sentences = len(sentences_from_regex)
|
|
|
words_per_sentence = [len(s.split()) for s in sentences_from_regex]
|
|
|
|
|
|
formal_words = ['Pan', 'Pani', 'Państwo', 'uprzejmie', 'proszę', 'dziękuję']
|
|
|
formal_count = sum(text.count(word) for word in formal_words)
|
|
|
|
|
|
cohesive_words = ['jednak', 'ponadto', 'w konsekwencji', 'zatem', 'więc', 'dlatego', 'natomiast', 'niemniej']
|
|
|
cohesion_count = sum(text.lower().count(word) for word in cohesive_words)
|
|
|
|
|
|
quote_count = len(re.findall(r'[„"]', text)) // 2
|
|
|
reference_count = len(re.findall(r'\[[0-9]+\]', text))
|
|
|
|
|
|
first_words = [s.split()[0].lower() for s in sentences_from_regex if s.split()]
|
|
|
|
|
|
return {
|
|
|
'formal_words_ratio': safe_divide(formal_count, len(words)),
|
|
|
'cohesive_words_per_sentence': safe_divide(cohesion_count, num_sentences),
|
|
|
'quotes_and_references_per_sentence': safe_divide(quote_count + reference_count, num_sentences),
|
|
|
'unique_sentence_beginnings_ratio': safe_divide(len(set(first_words)), num_sentences),
|
|
|
'commas_per_sentence': safe_divide(text.count(','), num_sentences),
|
|
|
'semicolons_per_sentence': safe_divide(text.count(';'), num_sentences),
|
|
|
'dashes_per_sentence': safe_divide(text.count('—') + text.count('–'), num_sentences),
|
|
|
'colons_per_sentence': safe_divide(text.count(':'), num_sentences),
|
|
|
'short_sentences_ratio': safe_divide(sum(1 for c in words_per_sentence if c < 5), num_sentences),
|
|
|
'long_sentences_ratio': safe_divide(sum(1 for c in words_per_sentence if c > 30), num_sentences)
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
def calculate_all_linguistic_features(text: str, text_lower: str, words: List[str], words_lower: List[str], sentences: List[str]) -> Dict[str, float]:
|
|
|
"""Agreguje wszystkie cechy lingwistyczne i stylistyczne z tego modułu."""
|
|
|
features = {}
|
|
|
features.update(calculate_stop_word_ratio(words_lower))
|
|
|
features.update(count_bad_words(words_lower))
|
|
|
features.update(calculate_unigram_entropy(words_lower))
|
|
|
features.update(count_non_alpha_words(text))
|
|
|
features.update(calculate_symbol_to_word_ratio(words, text))
|
|
|
features.update(calculate_ngram_fractions(words))
|
|
|
features.update(analyze_stylistic_metrics(text, words, sentences))
|
|
|
features['javascript_counts_per_line'] = text_lower.count('javascript')
|
|
|
return features |
