feature_extraction.py

import re
import numpy as np
import nltk
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.sentiment.vader import SentimentIntensityAnalyzer

# Deception-detection lexicons (LIWC-aligned)
HEDGE_WORDS = {'maybe', 'perhaps', 'possibly', 'might', 'could', 'seems',
               'apparently', 'supposedly', 'allegedly', 'reportedly', 'somewhat',
               'rather', 'quite', 'fairly', 'presumably'}

CERTAINTY_WORDS = {'definitely', 'certainly', 'absolutely', 'always', 'never',
                   'guaranteed', 'proven', 'undeniable', 'obvious', 'clearly',
                   'without doubt', 'of course', 'undoubtedly', 'surely'}

MODAL_VERBS = {'can', 'could', 'may', 'might', 'must', 'shall', 'should',
               'will', 'would'}

NEGATION_WORDS = {'not', "n't", 'no', 'never', 'neither', 'nor', 'none',
                  'nothing', 'nowhere', 'nobody', 'cannot', "don't", "doesn't",
                  "didn't", "won't", "isn't", "aren't", "wasn't", "weren't"}

EMOTIONAL_INTENSITY = {'shocking', 'outrageous', 'incredible', 'unbelievable',
                       'devastating', 'horrible', 'amazing', 'terrible',
                       'catastrophic', 'disgusting', 'explosive', 'disastrous'}

# Sensationalist / clickbait phrases common in fake news
SENSATIONAL_PHRASES = [
    'miracle cure', 'cures all', 'cure all', 'cures cancer', 'cure cancer',
    'secret cure', 'miracle drug', 'instant cure', 'instantly cures',
    'breakthrough cure', 'wonder drug', 'magic pill', 'one weird trick',
    'doctors hate', 'pharma doesn', 'big pharma', 'they don\'t want you',
    'what they don\'t tell you', 'the truth about', 'exposed',
    'banned by', 'cover-up', 'coverup', 'conspiracy',
]

# Conspiracy / misinformation language patterns
CONSPIRACY_PHRASES = [
    'secret documents', 'secret hospital', 'secret government',
    'secret plan', 'secretly adding', 'secret program',
    'control minds', 'mind control', 'control people',
    'chemtrails', 'flat earth', 'illuminati', 'new world order',
    'they are hiding', 'what they hide', 'hidden truth',
    'wake up', 'open your eyes', 'the real truth',
    'confirmed by secret', 'leaked documents', 'internal documents',
    'secretly control', 'control the population', 'government coverup',
    'adding chemicals', 'putting chemicals', 'chemical weapons',
    'control minds and', 'control peoples', 'control emotions',
]

# Extreme health / pseudoscience claims
HEALTH_MISINFO_PATTERNS = [
    'drinking bleach', 'eat bleach', 'bleach cure',
    'cures all types', 'cures every', 'cures 100',
    'within 48 hours', 'within 24 hours', 'overnight cure',
    'all types of cancer', 'all diseases', 'every disease',
    'no side effects', 'completely safe', '100 percent effective',
    'natural cure', 'home remedy cure', 'detox cleanse',
]

ATTRIBUTION_VERBS = {'said', 'claimed', 'stated', 'announced', 'reported',
                     'according', 'revealed', 'disclosed', 'alleged', 'insisted'}


class FeatureExtractor:
    def __init__(self):
        # Syntactic features (n-grams 1-3) per Shu et al. §3.2.1
        self.vectorizer = TfidfVectorizer(
            ngram_range=(1, 3), max_features=8000,
            analyzer='word', sublinear_tf=True
        )
        self.char_vectorizer = TfidfVectorizer(
            ngram_range=(2, 5), max_features=3000,
            analyzer='char_wb', sublinear_tf=True
        )
        for res in ['punkt', 'punkt_tab', 'averaged_perceptron_tagger', 'averaged_perceptron_tagger_eng', 'universal_tagset', 'vader_lexicon']:
            try:
                nltk.download(res, quiet=True)
            except Exception:
                pass
        try:
            self.sid = SentimentIntensityAnalyzer()
        except Exception:
            self.sid = None

    def count_syllables(self, word):
        word = word.lower()
        count = 0
        vowels = "aeiouy"
        if word[0] in vowels:
            count += 1
        for index in range(1, len(word)):
            if word[index] in vowels and word[index - 1] not in vowels:
                count += 1
        if word.endswith("e"):
            count -= 1
        if count == 0:
            count += 1
        return count

    def extract_content_features(self, text):
        """
        Extracts News Content Features (§3.2.1)
        Focuses on Lexical, Syntactic, and Style (Deception cues)
        """
        if not text:
            return {}

        tokens = nltk.word_tokenize(text)
        words = [w.lower() for w in tokens if w.isalnum()]
        total_words = len(words)
        unique_words = len(set(words))
        text_lower = text.lower()
        text_len = max(1, len(text))

        # 1. Lexical Features
        lexical = {
            'total_words': total_words,
            'lexical_density': unique_words / total_words if total_words > 0 else 0,
            'avg_word_length': np.mean([len(w) for w in words]) if words else 0,
            'capital_ratio': sum(1 for c in text if c.isupper()) / text_len
        }

        # 2. Syntactic & Style (POS Tagging)
        pos_tags_raw = nltk.pos_tag(tokens)
        pos_tags_univ = nltk.pos_tag(tokens, tagset='universal')
        tag_counts = {}
        for _, tag in pos_tags_univ:
            tag_counts[tag] = tag_counts.get(tag, 0) + 1

        total_tags = max(1, len(tokens))
        syntax = {
            'noun_ratio': tag_counts.get('NOUN', 0) / total_tags,
            'verb_ratio': tag_counts.get('VERB', 0) / total_tags,
            'adj_ratio': tag_counts.get('ADJ', 0) / total_tags,
            'adv_ratio': tag_counts.get('ADV', 0) / total_tags,
            'punctuation_aggression': sum(1 for char in text if char in '!') / text_len
        }

        # 3. NER & POS Trigrams (Linguistic Cadence)
        try:
            chunks = nltk.ne_chunk(pos_tags_raw)
            entities = [chunk for chunk in chunks if hasattr(chunk, 'label')]
            entity_density = len(entities) / max(1, total_words)
        except:
            entity_density = 0

        tags_only = [t for _, t in pos_tags_univ]
        trigrams = list(nltk.trigrams(tags_only))
        formal_markers = {('NOUN','VERB','NOUN'), ('ADJ','NOUN','VERB'), ('NOUN','ADP','NOUN')}
        formal_cadence = sum(1 for tr in trigrams if tr in formal_markers) / max(1, len(trigrams))

        # 4. Sentiment & Subjectivity
        sentiment = self.sid.polarity_scores(text)
        sentences = nltk.sent_tokenize(text)
        num_sentences = max(1, len(sentences))
        num_syllables = sum(self.count_syllables(w) for w in words)
        flesch = 206.835 - 1.015 * (total_words / num_sentences) - 84.6 * (num_syllables / max(1, total_words))
        subjectivity = (tag_counts.get('ADJ', 0) + tag_counts.get('ADV', 0)) / total_tags

        advanced = {
            'sentiment_score': sentiment['compound'],
            'complexity_score': flesch,
            'subjectivity_score': subjectivity,
            'entity_density': entity_density,
            'formal_cadence': formal_cadence,
            'official_marker': 1.0 if any(s in text.upper() for s in [
                'BUREAU OF', 'FEDERAL RESERVE', 'NOAA', 'STATISTICS REPORTED',
                'CENSUS BUREAU', 'WORLD HEALTH ORGANIZATION', 'PEER-REVIEWED',
                'PUBLISHED IN', 'ACCORDING TO DATA'
            ]) else 0.0
        }

        # 5. Style-based Deception Cues
        deception = {
            'exclamation_marks': text.count('!'),
            'question_marks': text.count('?'),
            'quotes_count': text.count('"') + text.count("'")
        }

        # ===== NEW: Deception Linguistic Features =====

        # 6. Hedging vs Certainty (deception often uses more certainty words)
        word_set = set(words)
        hedge_count = sum(1 for w in words if w in HEDGE_WORDS)
        certainty_count = sum(1 for w in words if w in CERTAINTY_WORDS)
        hedge_certainty = {
            'hedge_ratio': hedge_count / max(1, total_words),
            'certainty_ratio': certainty_count / max(1, total_words),
            'hedge_certainty_diff': (hedge_count - certainty_count) / max(1, total_words),
        }

        # 7. Emotional Intensity (fake news uses more emotional language)
        emotional_count = sum(1 for w in words if w in EMOTIONAL_INTENSITY)
        sentiment_abs = abs(sentiment['compound'])
        emotional = {
            'emotional_intensity': emotional_count / max(1, total_words),
            'sentiment_extremity': sentiment_abs,
            'negativity_score': abs(sentiment['neg']),
            'positivity_score': abs(sentiment['pos']),
        }

        # 8. Numerical Features (real news tends to have more specific numbers)
        numbers = re.findall(r'\b\d+\.?\d*\b', text)
        percentages = re.findall(r'\d+\s*%', text)
        dollar_amounts = re.findall(r'\$[\d,]+\.?\d*', text)
        numerical = {
            'number_density': len(numbers) / max(1, total_words),
            'has_percentage': 1.0 if percentages else 0.0,
            'has_dollar_amount': 1.0 if dollar_amounts else 0.0,
            'number_count': len(numbers),
        }

        # 9. Attribution & Source Cues (real news cites sources)
        attribution_count = sum(1 for w in words if w in ATTRIBUTION_VERBS)
        has_source = 1.0 if any(p in text_lower for p in [
            'according to', 'studies show', 'research suggests',
            'data from', 'report by', 'analysis by'
        ]) else 0.0
        source = {
            'attribution_ratio': attribution_count / max(1, total_words),
            'has_source_citation': has_source,
        }

        # 10. Readability (Automated Readability Index)
        char_count = sum(1 for c in text if c.isalnum())
        ari = (4.71 * char_count / max(1, total_words)) + (0.5 * total_words / num_sentences) - 21.43
        gunning_fog = 0.4 * ((total_words / num_sentences) + 100 * (
            sum(1 for w in words if self.count_syllables(w) >= 3) / max(1, total_words)))
        readability = {
            'ari_score': ari,
            'gunning_fog': gunning_fog,
            'avg_sentence_length': total_words / num_sentences,
        }

        # 11. Pronoun & Modality Features
        first_person = sum(1 for w in words if w in {'i', 'me', 'my', 'mine', 'we', 'our', 'ours', 'myself'})
        modal_count = sum(1 for w in words if w in MODAL_VERBS)
        negation_count = sum(1 for w in words if w in NEGATION_WORDS or w.endswith("n't"))
        pronoun_modality = {
            'first_person_ratio': first_person / max(1, total_words),
            'modal_ratio': modal_count / max(1, total_words),
            'negation_ratio': negation_count / max(1, total_words),
        }

        # 12. Comparative & Superlative markers
        comparative = sum(1 for _, t in pos_tags_raw if t in ('JJR', 'RBR'))
        superlative = sum(1 for _, t in pos_tags_raw if t in ('JJS', 'RBS'))
        comparison = {
            'comparative_ratio': comparative / max(1, total_tags),
            'superlative_ratio': superlative / max(1, total_tags),
        }

        # 13. Passive voice detection (be + VBN patterns)
        passive_patterns = sum(1 for i in range(len(pos_tags_raw) - 1)
                               if pos_tags_raw[i][1] in ('VBZ', 'VBP', 'VBD', 'VBN')
                               and pos_tags_raw[i+1][1] == 'VBN')
        voice = {
            'passive_ratio': passive_patterns / max(1, total_tags),
        }

        # 14. ALL-CAPS word ratio (shouting / urgency signal)
        caps_words = sum(1 for w in words if w.isupper() and len(w) > 2)
        urgency = {
            'caps_word_ratio': caps_words / max(1, total_words),
            'ellipsis_count': text.count('...'),
        }

        # 15. Sensationalism & Conspiracy Score (fake news signature)
        text_lower_joined = ' ' + text_lower + ' '
        sensational_hits = sum(1 for p in SENSATIONAL_PHRASES if p in text_lower_joined)
        conspiracy_hits = sum(1 for p in CONSPIRACY_PHRASES if p in text_lower_joined)
        health_misinfo_hits = sum(1 for p in HEALTH_MISINFO_PATTERNS if p in text_lower_joined)
        sensationalism = {
            'sensationalism_score': (sensational_hits + conspiracy_hits + health_misinfo_hits) / max(1, total_words),
            'conspiracy_score': conspiracy_hits / max(1, total_words),
            'health_misinfo_score': health_misinfo_hits / max(1, total_words),
            'sensational_hit_count': sensational_hits + conspiracy_hits + health_misinfo_hits,
        }

        # 16. Absolutist language (ALL, EVERY, NEVER — common in fake news)
        absolutist_words = sum(1 for w in words if w in {
            'all', 'every', 'always', 'never', 'everyone', 'nobody',
            'nothing', 'completely', 'totally', 'absolutely', 'entirely',
            '100', 'instantly', 'instant', 'guaranteed',
        })
        extremity = {
            'absolutist_ratio': absolutist_words / max(1, total_words),
        }

        return {
            **lexical, **syntax, **advanced, **deception,
            **hedge_certainty, **emotional, **numerical,
            **source, **readability, **pronoun_modality,
            **comparison, **voice, **urgency,
            **sensationalism, **extremity
        }

    def get_combined_features(self, text, metadata=None):
        """
        Combines News Content features with Social Context proxies (§3.2.2)
        """
        features = self.extract_content_features(text)

        if metadata:
            # AUXILIARY INFORMATION (Social Context Proxy §3.2.2)
            history_cols = ['barely_true_counts', 'false_counts', 'half_true_counts',
                            'mostly_true_counts', 'pants_on_fire_counts']

            total_history = sum(float(metadata.get(c, 0)) for c in history_cols)
            reliable_history = float(metadata.get('half_true_counts', 0)) + \
                               float(metadata.get('mostly_true_counts', 0))

            features['speaker_reliability'] = reliable_history / total_history if total_history > 0 else 0.5

            # Publisher Context (§3.1 distortion bias)
            features['is_republican'] = 1 if metadata.get('party') == 'republican' else 0
            features['is_democrat'] = 1 if metadata.get('party') == 'democrat' else 0

            # NEW: Historical track record features
            false_total = float(metadata.get('false_counts', 0)) + float(metadata.get('pants_on_fire_counts', 0))
            true_total = float(metadata.get('mostly_true_counts', 0)) + float(metadata.get('half_true_counts', 0))
            features['false_history_ratio'] = false_total / max(1, total_history)
            features['true_history_ratio'] = true_total / max(1, total_history)
            features['history_volume'] = np.log1p(total_history)  # log-scaled volume

        return features

    def transform_text_tfidf(self, corpus):
        return self.vectorizer.fit_transform(corpus)

    def transform_char_tfidf(self, corpus):
        return self.char_vectorizer.fit_transform(corpus)