Upload feature_extraction.py

feature_extraction.py

@@ -0,0 +1,333 @@
+import re
+import numpy as np
+import nltk
+from sklearn.feature_extraction.text import TfidfVectorizer
+from nltk.sentiment.vader import SentimentIntensityAnalyzer
+
+# Deception-detection lexicons (LIWC-aligned)
+HEDGE_WORDS = {'maybe', 'perhaps', 'possibly', 'might', 'could', 'seems',
+               'apparently', 'supposedly', 'allegedly', 'reportedly', 'somewhat',
+               'rather', 'quite', 'fairly', 'presumably'}
+
+CERTAINTY_WORDS = {'definitely', 'certainly', 'absolutely', 'always', 'never',
+                   'guaranteed', 'proven', 'undeniable', 'obvious', 'clearly',
+                   'without doubt', 'of course', 'undoubtedly', 'surely'}
+
+MODAL_VERBS = {'can', 'could', 'may', 'might', 'must', 'shall', 'should',
+               'will', 'would'}
+
+NEGATION_WORDS = {'not', "n't", 'no', 'never', 'neither', 'nor', 'none',
+                  'nothing', 'nowhere', 'nobody', 'cannot', "don't", "doesn't",
+                  "didn't", "won't", "isn't", "aren't", "wasn't", "weren't"}
+
+EMOTIONAL_INTENSITY = {'shocking', 'outrageous', 'incredible', 'unbelievable',
+                       'devastating', 'horrible', 'amazing', 'terrible',
+                       'catastrophic', 'disgusting', 'explosive', 'disastrous'}
+
+# Sensationalist / clickbait phrases common in fake news
+SENSATIONAL_PHRASES = [
+    'miracle cure', 'cures all', 'cure all', 'cures cancer', 'cure cancer',
+    'secret cure', 'miracle drug', 'instant cure', 'instantly cures',
+    'breakthrough cure', 'wonder drug', 'magic pill', 'one weird trick',
+    'doctors hate', 'pharma doesn', 'big pharma', 'they don\'t want you',
+    'what they don\'t tell you', 'the truth about', 'exposed',
+    'banned by', 'cover-up', 'coverup', 'conspiracy',
+]
+
+# Conspiracy / misinformation language patterns
+CONSPIRACY_PHRASES = [
+    'secret documents', 'secret hospital', 'secret government',
+    'secret plan', 'secretly adding', 'secret program',
+    'control minds', 'mind control', 'control people',
+    'chemtrails', 'flat earth', 'illuminati', 'new world order',
+    'they are hiding', 'what they hide', 'hidden truth',
+    'wake up', 'open your eyes', 'the real truth',
+    'confirmed by secret', 'leaked documents', 'internal documents',
+    'secretly control', 'control the population', 'government coverup',
+    'adding chemicals', 'putting chemicals', 'chemical weapons',
+    'control minds and', 'control peoples', 'control emotions',
+]
+
+# Extreme health / pseudoscience claims
+HEALTH_MISINFO_PATTERNS = [
+    'drinking bleach', 'eat bleach', 'bleach cure',
+    'cures all types', 'cures every', 'cures 100',
+    'within 48 hours', 'within 24 hours', 'overnight cure',
+    'all types of cancer', 'all diseases', 'every disease',
+    'no side effects', 'completely safe', '100 percent effective',
+    'natural cure', 'home remedy cure', 'detox cleanse',
+]
+
+ATTRIBUTION_VERBS = {'said', 'claimed', 'stated', 'announced', 'reported',
+                     'according', 'revealed', 'disclosed', 'alleged', 'insisted'}
+
+
+class FeatureExtractor:
+    def __init__(self):
+        # Syntactic features (n-grams 1-3) per Shu et al. §3.2.1
+        self.vectorizer = TfidfVectorizer(
+            ngram_range=(1, 3), max_features=8000,
+            analyzer='word', sublinear_tf=True
+        )
+        self.char_vectorizer = TfidfVectorizer(
+            ngram_range=(2, 5), max_features=3000,
+            analyzer='char_wb', sublinear_tf=True
+        )
+        for res in ['punkt', 'punkt_tab', 'averaged_perceptron_tagger', 'averaged_perceptron_tagger_eng', 'universal_tagset', 'vader_lexicon']:
+            try:
+                nltk.download(res, quiet=True)
+            except Exception:
+                pass
+        try:
+            self.sid = SentimentIntensityAnalyzer()
+        except Exception:
+            self.sid = None
+
+    def count_syllables(self, word):
+        word = word.lower()
+        count = 0
+        vowels = "aeiouy"
+        if word[0] in vowels:
+            count += 1
+        for index in range(1, len(word)):
+            if word[index] in vowels and word[index - 1] not in vowels:
+                count += 1
+        if word.endswith("e"):
+            count -= 1
+        if count == 0:
+            count += 1
+        return count
+
+    def extract_content_features(self, text):
+        """
+        Extracts News Content Features (§3.2.1)
+        Focuses on Lexical, Syntactic, and Style (Deception cues)
+        """
+        if not text:
+            return {}
+
+        tokens = nltk.word_tokenize(text)
+        words = [w.lower() for w in tokens if w.isalnum()]
+        total_words = len(words)
+        unique_words = len(set(words))
+        text_lower = text.lower()
+        text_len = max(1, len(text))
+
+        # 1. Lexical Features
+        lexical = {
+            'total_words': total_words,
+            'lexical_density': unique_words / total_words if total_words > 0 else 0,
+            'avg_word_length': np.mean([len(w) for w in words]) if words else 0,
+            'capital_ratio': sum(1 for c in text if c.isupper()) / text_len
+        }
+
+        # 2. Syntactic & Style (POS Tagging)
+        pos_tags_raw = nltk.pos_tag(tokens)
+        pos_tags_univ = nltk.pos_tag(tokens, tagset='universal')
+        tag_counts = {}
+        for _, tag in pos_tags_univ:
+            tag_counts[tag] = tag_counts.get(tag, 0) + 1
+
+        total_tags = max(1, len(tokens))
+        syntax = {
+            'noun_ratio': tag_counts.get('NOUN', 0) / total_tags,
+            'verb_ratio': tag_counts.get('VERB', 0) / total_tags,
+            'adj_ratio': tag_counts.get('ADJ', 0) / total_tags,
+            'adv_ratio': tag_counts.get('ADV', 0) / total_tags,
+            'punctuation_aggression': sum(1 for char in text if char in '!') / text_len
+        }
+
+        # 3. NER & POS Trigrams (Linguistic Cadence)
+        try:
+            chunks = nltk.ne_chunk(pos_tags_raw)
+            entities = [chunk for chunk in chunks if hasattr(chunk, 'label')]
+            entity_density = len(entities) / max(1, total_words)
+        except:
+            entity_density = 0
+
+        tags_only = [t for _, t in pos_tags_univ]
+        trigrams = list(nltk.trigrams(tags_only))
+        formal_markers = {('NOUN','VERB','NOUN'), ('ADJ','NOUN','VERB'), ('NOUN','ADP','NOUN')}
+        formal_cadence = sum(1 for tr in trigrams if tr in formal_markers) / max(1, len(trigrams))
+
+        # 4. Sentiment & Subjectivity
+        sentiment = self.sid.polarity_scores(text)
+        sentences = nltk.sent_tokenize(text)
+        num_sentences = max(1, len(sentences))
+        num_syllables = sum(self.count_syllables(w) for w in words)
+        flesch = 206.835 - 1.015 * (total_words / num_sentences) - 84.6 * (num_syllables / max(1, total_words))
+        subjectivity = (tag_counts.get('ADJ', 0) + tag_counts.get('ADV', 0)) / total_tags
+
+        advanced = {
+            'sentiment_score': sentiment['compound'],
+            'complexity_score': flesch,
+            'subjectivity_score': subjectivity,
+            'entity_density': entity_density,
+            'formal_cadence': formal_cadence,
+            'official_marker': 1.0 if any(s in text.upper() for s in [
+                'BUREAU OF', 'FEDERAL RESERVE', 'NOAA', 'STATISTICS REPORTED',
+                'CENSUS BUREAU', 'WORLD HEALTH ORGANIZATION', 'PEER-REVIEWED',
+                'PUBLISHED IN', 'ACCORDING TO DATA'
+            ]) else 0.0
+        }
+
+        # 5. Style-based Deception Cues
+        deception = {
+            'exclamation_marks': text.count('!'),
+            'question_marks': text.count('?'),
+            'quotes_count': text.count('"') + text.count("'")
+        }
+
+        # ===== NEW: Deception Linguistic Features =====
+
+        # 6. Hedging vs Certainty (deception often uses more certainty words)
+        word_set = set(words)
+        hedge_count = sum(1 for w in words if w in HEDGE_WORDS)
+        certainty_count = sum(1 for w in words if w in CERTAINTY_WORDS)
+        hedge_certainty = {
+            'hedge_ratio': hedge_count / max(1, total_words),
+            'certainty_ratio': certainty_count / max(1, total_words),
+            'hedge_certainty_diff': (hedge_count - certainty_count) / max(1, total_words),
+        }
+
+        # 7. Emotional Intensity (fake news uses more emotional language)
+        emotional_count = sum(1 for w in words if w in EMOTIONAL_INTENSITY)
+        sentiment_abs = abs(sentiment['compound'])
+        emotional = {
+            'emotional_intensity': emotional_count / max(1, total_words),
+            'sentiment_extremity': sentiment_abs,
+            'negativity_score': abs(sentiment['neg']),
+            'positivity_score': abs(sentiment['pos']),
+        }
+
+        # 8. Numerical Features (real news tends to have more specific numbers)
+        numbers = re.findall(r'\b\d+\.?\d*\b', text)
+        percentages = re.findall(r'\d+\s*%', text)
+        dollar_amounts = re.findall(r'\$[\d,]+\.?\d*', text)
+        numerical = {
+            'number_density': len(numbers) / max(1, total_words),
+            'has_percentage': 1.0 if percentages else 0.0,
+            'has_dollar_amount': 1.0 if dollar_amounts else 0.0,
+            'number_count': len(numbers),
+        }
+
+        # 9. Attribution & Source Cues (real news cites sources)
+        attribution_count = sum(1 for w in words if w in ATTRIBUTION_VERBS)
+        has_source = 1.0 if any(p in text_lower for p in [
+            'according to', 'studies show', 'research suggests',
+            'data from', 'report by', 'analysis by'
+        ]) else 0.0
+        source = {
+            'attribution_ratio': attribution_count / max(1, total_words),
+            'has_source_citation': has_source,
+        }
+
+        # 10. Readability (Automated Readability Index)
+        char_count = sum(1 for c in text if c.isalnum())
+        ari = (4.71 * char_count / max(1, total_words)) + (0.5 * total_words / num_sentences) - 21.43
+        gunning_fog = 0.4 * ((total_words / num_sentences) + 100 * (
+            sum(1 for w in words if self.count_syllables(w) >= 3) / max(1, total_words)))
+        readability = {
+            'ari_score': ari,
+            'gunning_fog': gunning_fog,
+            'avg_sentence_length': total_words / num_sentences,
+        }
+
+        # 11. Pronoun & Modality Features
+        first_person = sum(1 for w in words if w in {'i', 'me', 'my', 'mine', 'we', 'our', 'ours', 'myself'})
+        modal_count = sum(1 for w in words if w in MODAL_VERBS)
+        negation_count = sum(1 for w in words if w in NEGATION_WORDS or w.endswith("n't"))
+        pronoun_modality = {
+            'first_person_ratio': first_person / max(1, total_words),
+            'modal_ratio': modal_count / max(1, total_words),
+            'negation_ratio': negation_count / max(1, total_words),
+        }
+
+        # 12. Comparative & Superlative markers
+        comparative = sum(1 for _, t in pos_tags_raw if t in ('JJR', 'RBR'))
+        superlative = sum(1 for _, t in pos_tags_raw if t in ('JJS', 'RBS'))
+        comparison = {
+            'comparative_ratio': comparative / max(1, total_tags),
+            'superlative_ratio': superlative / max(1, total_tags),
+        }
+
+        # 13. Passive voice detection (be + VBN patterns)
+        passive_patterns = sum(1 for i in range(len(pos_tags_raw) - 1)
+                               if pos_tags_raw[i][1] in ('VBZ', 'VBP', 'VBD', 'VBN')
+                               and pos_tags_raw[i+1][1] == 'VBN')
+        voice = {
+            'passive_ratio': passive_patterns / max(1, total_tags),
+        }
+
+        # 14. ALL-CAPS word ratio (shouting / urgency signal)
+        caps_words = sum(1 for w in words if w.isupper() and len(w) > 2)
+        urgency = {
+            'caps_word_ratio': caps_words / max(1, total_words),
+            'ellipsis_count': text.count('...'),
+        }
+
+        # 15. Sensationalism & Conspiracy Score (fake news signature)
+        text_lower_joined = ' ' + text_lower + ' '
+        sensational_hits = sum(1 for p in SENSATIONAL_PHRASES if p in text_lower_joined)
+        conspiracy_hits = sum(1 for p in CONSPIRACY_PHRASES if p in text_lower_joined)
+        health_misinfo_hits = sum(1 for p in HEALTH_MISINFO_PATTERNS if p in text_lower_joined)
+        sensationalism = {
+            'sensationalism_score': (sensational_hits + conspiracy_hits + health_misinfo_hits) / max(1, total_words),
+            'conspiracy_score': conspiracy_hits / max(1, total_words),
+            'health_misinfo_score': health_misinfo_hits / max(1, total_words),
+            'sensational_hit_count': sensational_hits + conspiracy_hits + health_misinfo_hits,
+        }
+
+        # 16. Absolutist language (ALL, EVERY, NEVER — common in fake news)
+        absolutist_words = sum(1 for w in words if w in {
+            'all', 'every', 'always', 'never', 'everyone', 'nobody',
+            'nothing', 'completely', 'totally', 'absolutely', 'entirely',
+            '100', 'instantly', 'instant', 'guaranteed',
+        })
+        extremity = {
+            'absolutist_ratio': absolutist_words / max(1, total_words),
+        }
+
+        return {
+            **lexical, **syntax, **advanced, **deception,
+            **hedge_certainty, **emotional, **numerical,
+            **source, **readability, **pronoun_modality,
+            **comparison, **voice, **urgency,
+            **sensationalism, **extremity
+        }
+
+    def get_combined_features(self, text, metadata=None):
+        """
+        Combines News Content features with Social Context proxies (§3.2.2)
+        """
+        features = self.extract_content_features(text)
+
+        if metadata:
+            # AUXILIARY INFORMATION (Social Context Proxy §3.2.2)
+            history_cols = ['barely_true_counts', 'false_counts', 'half_true_counts',
+                            'mostly_true_counts', 'pants_on_fire_counts']
+
+            total_history = sum(float(metadata.get(c, 0)) for c in history_cols)
+            reliable_history = float(metadata.get('half_true_counts', 0)) + \
+                               float(metadata.get('mostly_true_counts', 0))
+
+            features['speaker_reliability'] = reliable_history / total_history if total_history > 0 else 0.5
+
+            # Publisher Context (§3.1 distortion bias)
+            features['is_republican'] = 1 if metadata.get('party') == 'republican' else 0
+            features['is_democrat'] = 1 if metadata.get('party') == 'democrat' else 0
+
+            # NEW: Historical track record features
+            false_total = float(metadata.get('false_counts', 0)) + float(metadata.get('pants_on_fire_counts', 0))
+            true_total = float(metadata.get('mostly_true_counts', 0)) + float(metadata.get('half_true_counts', 0))
+            features['false_history_ratio'] = false_total / max(1, total_history)
+            features['true_history_ratio'] = true_total / max(1, total_history)
+            features['history_volume'] = np.log1p(total_history)  # log-scaled volume
+
+        return features
+
+    def transform_text_tfidf(self, corpus):
+        return self.vectorizer.fit_transform(corpus)
+
+    def transform_char_tfidf(self, corpus):
+        return self.char_vectorizer.fit_transform(corpus)