extractor.py

# core/extractor.py
import re
import hashlib


class FactExtractor:
    # Simple sentence splitter: split on .!? followed by space and a capital letter.
    # We skip the variable-width lookbehind which causes the re.error.
    _SENTENCE_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-Z])')

    def extract_facts(self, text, topic=""):
        """Extract high-quality, atomic fact sentences from raw text."""
        if not text or not text.strip():
            return []

        # Clean text
        text = re.sub(r"\s+", " ", text).strip()
        text = re.sub(r"[^\x20-\x7E\n]", " ", text)  # Remove non-ASCII junk

        # Split into sentences
        sentences = self._SENTENCE_SPLIT.split(text)

        topic_lower = topic.lower()
        scored = []
        seen_hashes = set()

        for s in sentences:
            s = s.strip()
            if not s:
                continue

            words = s.split()
            word_count = len(words)

            # Filter by length
            if not (8 <= word_count <= 120):
                continue

            # Skip sentences that look like navigation/boilerplate
            if self._is_boilerplate(s):
                continue

            # Deduplicate by content hash (first 80 chars)
            fingerprint = hashlib.md5(s[:80].lower().encode()).hexdigest()
            if fingerprint in seen_hashes:
                continue
            seen_hashes.add(fingerprint)

            # Score: higher if contains topic keyword
            score = self._score_sentence(s, topic_lower, word_count)
            scored.append((score, s))

        # Sort by score (highest first), return top 20
        scored.sort(key=lambda x: x[0], reverse=True)
        return [s for _, s in scored[:20]]

    def _score_sentence(self, sentence, topic, word_count):
        """Score a sentence by information density and topic relevance."""
        score = 0.0
        s_lower = sentence.lower()

        # Topic match
        if topic and topic in s_lower:
            score += 3.0

        # Contains numbers/data (more factual)
        if re.search(r"\d", sentence):
            score += 1.5

        # Contains proper nouns (capitalized words not at sentence start)
        proper_nouns = re.findall(r"(?<!\. )\b[A-Z][a-z]{2,}\b", sentence[5:])
        score += min(len(proper_nouns) * 0.5, 2.0)

        # Penalize very short or very long
        if word_count < 12:
            score -= 0.5
        if word_count > 80:
            score -= 1.0

        # Penalize question sentences (less factual)
        if sentence.endswith("?"):
            score -= 1.0

        return score

    def _is_boilerplate(self, sentence):
        """Detect navigation, cookie notices, ads, etc."""
        s_lower = sentence.lower()
        boilerplate_phrases = [
            "cookie", "privacy policy", "terms of service", "click here",
            "subscribe", "newsletter", "all rights reserved", "copyright ©",
            "sign up", "log in", "login", "register now", "follow us",
            "share this", "read more", "learn more →", "advertisement",
            "sponsored", "skip to content", "back to top",
        ]
        if any(p in s_lower for p in boilerplate_phrases):
            return True

        # All caps (likely a heading or ad)
        if sentence.isupper() and len(sentence) > 10:
            return True

        # Very few unique words (repetitive/spammy)
        words = sentence.lower().split()
        if len(words) > 5 and len(set(words)) / len(words) < 0.4:
            return True

        return False