File size: 3,573 Bytes
3d7a63c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | # core/extractor.py
import re
import hashlib
class FactExtractor:
# Simple sentence splitter: split on .!? followed by space and a capital letter.
# We skip the variable-width lookbehind which causes the re.error.
_SENTENCE_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-Z])')
def extract_facts(self, text, topic=""):
"""Extract high-quality, atomic fact sentences from raw text."""
if not text or not text.strip():
return []
# Clean text
text = re.sub(r"\s+", " ", text).strip()
text = re.sub(r"[^\x20-\x7E\n]", " ", text) # Remove non-ASCII junk
# Split into sentences
sentences = self._SENTENCE_SPLIT.split(text)
topic_lower = topic.lower()
scored = []
seen_hashes = set()
for s in sentences:
s = s.strip()
if not s:
continue
words = s.split()
word_count = len(words)
# Filter by length
if not (8 <= word_count <= 120):
continue
# Skip sentences that look like navigation/boilerplate
if self._is_boilerplate(s):
continue
# Deduplicate by content hash (first 80 chars)
fingerprint = hashlib.md5(s[:80].lower().encode()).hexdigest()
if fingerprint in seen_hashes:
continue
seen_hashes.add(fingerprint)
# Score: higher if contains topic keyword
score = self._score_sentence(s, topic_lower, word_count)
scored.append((score, s))
# Sort by score (highest first), return top 20
scored.sort(key=lambda x: x[0], reverse=True)
return [s for _, s in scored[:20]]
def _score_sentence(self, sentence, topic, word_count):
"""Score a sentence by information density and topic relevance."""
score = 0.0
s_lower = sentence.lower()
# Topic match
if topic and topic in s_lower:
score += 3.0
# Contains numbers/data (more factual)
if re.search(r"\d", sentence):
score += 1.5
# Contains proper nouns (capitalized words not at sentence start)
proper_nouns = re.findall(r"(?<!\. )\b[A-Z][a-z]{2,}\b", sentence[5:])
score += min(len(proper_nouns) * 0.5, 2.0)
# Penalize very short or very long
if word_count < 12:
score -= 0.5
if word_count > 80:
score -= 1.0
# Penalize question sentences (less factual)
if sentence.endswith("?"):
score -= 1.0
return score
def _is_boilerplate(self, sentence):
"""Detect navigation, cookie notices, ads, etc."""
s_lower = sentence.lower()
boilerplate_phrases = [
"cookie", "privacy policy", "terms of service", "click here",
"subscribe", "newsletter", "all rights reserved", "copyright ©",
"sign up", "log in", "login", "register now", "follow us",
"share this", "read more", "learn more →", "advertisement",
"sponsored", "skip to content", "back to top",
]
if any(p in s_lower for p in boilerplate_phrases):
return True
# All caps (likely a heading or ad)
if sentence.isupper() and len(sentence) > 10:
return True
# Very few unique words (repetitive/spammy)
words = sentence.lower().split()
if len(words) > 5 and len(set(words)) / len(words) < 0.4:
return True
return False |