Upload 2 files

chatbot.py

@@ -0,0 +1,212 @@
+"""
+chatbot.py — Answers naturally from what the AI has learned.
+Filters out junk, synthesizes real paragraph answers.
+"""
+import json, re, math, os, random
+from collections import Counter, defaultdict
+from html import unescape
+
+KNOWLEDGE_FILE = 'knowledge.jsonl'
+
+STOPWORDS = {
+    'a','an','the','is','it','in','on','at','to','for','of','and','or','but',
+    'was','are','were','be','been','have','has','had','do','does','did','will',
+    'would','could','should','may','might','that','this','these','those','with',
+    'from','by','as','not','also','than','then','so','if','when','what','how',
+    'who','which','its','their','our','your','my','his','her','we','they','he',
+    'she','you','i','me','him','us','them','said','says',
+}
+
+# Sentences that are website UI / taglines — not real knowledge
+JUNK = re.compile(
+    r'(sign up|subscribe|click here|breaking news:|stay informed|'
+    r'all rights reserved|privacy policy|cookie policy|advertisement|'
+    r'follow us|share this|read more|learn more|'
+    r'has everything you need|stay up to date|get the latest|'
+    r'news today ap|ap news|everything you need to know for|'
+    r'top breaking|live science\.|sciencedaily|newscientist|'
+    r'latest \w+ news articles today)',
+    re.IGNORECASE
+)
+
+def tokenize(text):
+    text = re.sub(r'[^\w\s]', ' ', text.lower())
+    return [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
+
+def scrub(text):
+    """Clean HTML artifacts from stored text."""
+    text = unescape(str(text))
+    text = re.sub(r'&#?[a-zA-Z0-9]+;', ' ', text)
+    text = re.sub(r'<[^>]+>', ' ', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def is_junk(sentence):
+    if len(sentence.split()) < 7: return True
+    if JUNK.search(sentence): return True
+    return False
+
+
+class KnowledgeBase:
+    def __init__(self):
+        self.docs=[]; self.doc_tokens=[]; self.idf={}; self.last_count=0
+        self._load()
+
+    def _load(self):
+        if not os.path.exists(KNOWLEDGE_FILE): return
+        new_docs = []
+        try:
+            with open(KNOWLEDGE_FILE,'r',encoding='utf-8') as f:
+                for line in f:
+                    l=line.strip()
+                    if l:
+                        try: new_docs.append(json.loads(l))
+                        except: pass
+        except: return
+        if len(new_docs)==self.last_count: return
+        self.docs=new_docs; self.last_count=len(new_docs)
+        self.doc_tokens=[tokenize(scrub(d.get('text',''))) for d in self.docs]
+        N=len(self.docs)
+        df=defaultdict(int)
+        for tok in self.doc_tokens:
+            for w in set(tok): df[w]+=1
+        self.idf={w:math.log((N+1)/(c+1))+1 for w,c in df.items()}
+
+    def search(self, query, top_k=10):
+        self._load()
+        if not self.docs: return []
+        q=tokenize(query)
+        if not q: return []
+        def sc(dt):
+            tf=Counter(dt); n=max(len(dt),1)
+            return sum((tf[w]/n)*self.idf.get(w,0) for w in q if w in tf)
+        ranked=sorted(range(len(self.docs)),key=lambda i:-sc(self.doc_tokens[i]))
+        return [(self.docs[i], sc(self.doc_tokens[i]))
+                for i in ranked[:top_k] if sc(self.doc_tokens[i])>0]
+
+    def get_stats(self):
+        self._load()
+        return {
+            'total':      len(self.docs),
+            'categories': dict(Counter(d.get('category','other') for d in self.docs)),
+            'sources':    dict(Counter(d.get('source','unknown') for d in self.docs)),
+        }
+
+
+def build_answer(query, results):
+    """
+    Pull the best real sentences from retrieved docs and write a natural answer.
+    Skips junk/tagline sentences. Returns None if nothing good found.
+    """
+    q_tokens = set(tokenize(query))
+    candidates = []
+
+    for doc, relevance in results:
+        text = scrub(doc.get('text',''))
+        # Split into sentences
+        sents = re.split(r'(?<=[.!?])\s+', text)
+        for s in sents:
+            s = s.strip()
+            if is_junk(s): continue
+            overlap = len(q_tokens & set(tokenize(s)))
+            # Score = topic overlap + doc relevance
+            candidates.append((overlap * 2 + relevance, s))
+
+    if not candidates:
+        return None
+
+    # Deduplicate, pick top unique sentences
+    seen = set(); good = []
+    for score, sent in sorted(candidates, reverse=True):
+        key = ' '.join(sent.lower().split()[:6])
+        if key not in seen:
+            seen.add(key)
+            good.append(sent)
+        if len(good) >= 5: break
+
+    if not good:
+        return None
+
+    # Write as flowing paragraphs (not bullets)
+    if len(good) == 1:
+        return good[0]
+    elif len(good) <= 3:
+        return ' '.join(good)
+    else:
+        # Split into two paragraphs
+        return ' '.join(good[:2]) + '\n\n' + ' '.join(good[2:])
+
+
+class RAGChatbot:
+    def __init__(self):
+        self.kb = KnowledgeBase()
+
+    def _get_reply(self, query, results, stats):
+        total = stats['total']
+
+        if total == 0:
+            return ("My brain is empty right now 🧠\n\n"
+                    "The app is already auto-training — just wait a minute "
+                    "while I read some articles, then ask again!")
+
+        answer = build_answer(query, results) if results else None
+
+        if not answer:
+            topics = ', '.join(list(stats['categories'].keys())[:5])
+            return (f"I've read **{total} articles** but haven't learned enough "
+                    f"about *\"{query}\"* yet.\n\n"
+                    f"I know most about: **{topics}**.\n\n"
+                    f"Keep me running and I'll learn more!")
+
+        return answer
+
+    def chat(self, user_message, history):
+        history = list(history or [])
+        msg = user_message.strip()
+        if not msg: return history
+
+        lower = msg.lower()
+        stats = self.kb.get_stats()
+
+        if lower in ('hi','hello','hey','sup','yo','hiya','howdy','helo','hai'):
+            total = stats['total']
+            if total == 0:
+                bot_reply = "Hey! 👋 I'm just starting up — give me a minute to read some articles and I'll be ready to answer questions!"
+            else:
+                topics = ', '.join(list(stats['categories'].keys())[:4])
+                bot_reply = f"Hey! 👋 I've read **{total} articles** so far. I know quite a bit about **{topics}**. What do you want to know?"
+
+        elif any(p in lower for p in ('how are you',"what's up",'whats up')):
+            bot_reply = "I'm doing great, always learning! 🤖 What do you want to know?"
+
+        elif any(p in lower for p in ('who are you','what are you','what can you do')):
+            bot_reply = (
+                "I'm a **Living Neural Network** — an AI that only knows what it has actually read. 🧠\n\n"
+                f"So far I've read **{stats['total']} articles** from the internet. "
+                "I don't have any pre-built knowledge — everything I know came from real articles I've been fed. "
+                "The more I train, the smarter I get. Ask me anything!"
+            )
+
+        elif lower in ('stats','status','knowledge'):
+            lines = '\n'.join(f"• **{k}**: {v} articles"
+                             for k,v in sorted(stats['categories'].items(), key=lambda x:-x[1]))
+            srcs  = ', '.join(f"{k} ({v})" for k,v in list(stats['sources'].items())[:6])
+            bot_reply = (f"**What I've learned:**\n\n{lines or '• nothing yet'}\n\n"
+                         f"**Sources:** {srcs or 'none yet'}\n"
+                         f"**Total:** {stats['total']} articles")
+
+        elif lower in ('help','?'):
+            bot_reply = ("Just ask me anything — I'll answer from what I've actually read!\n\n"
+                         "Try: *What is science?* · *Who is Elon Musk?* · *What's happening in AI?*\n\n"
+                         "Type `stats` to see what I know.")
+
+        elif any(p in lower for p in ('thanks','thank you','thx','ty')):
+            bot_reply = "You're welcome! 😊"
+
+        else:
+            results   = self.kb.search(msg, top_k=10)
+            bot_reply = self._get_reply(msg, results, stats)
+
+        history.append({"role": "user",      "content": user_message})
+        history.append({"role": "assistant", "content": bot_reply})
+        return history

data_fetcher.py

@@ -0,0 +1,270 @@
+"""
+data_fetcher.py — Fetches REAL article content (not website taglines).
+Key fix: extracts full paragraph text from pages, filters out metadata garbage.
+"""
+import requests, feedparser, random, re, time, json
+from datetime import datetime
+from collections import deque
+from urllib.parse import quote_plus
+from html import unescape
+
+try:
+    from bs4 import BeautifulSoup
+    BS4 = True
+except ImportError:
+    BS4 = False
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+}
+
+# ── GARBAGE PATTERNS — these are website UI text, not real content ─────────────
+GARBAGE_PATTERNS = re.compile(
+    r'(sign up|subscribe|click here|breaking news|stay informed|'
+    r'all rights reserved|privacy policy|cookie|advertisement|'
+    r'newsletter|follow us|share this|read more|learn more|'
+    r'top stories|latest news from|everything you need to know|'
+    r'has everything|stay up to date|get the latest|'
+    r'news articles today|news today ap|ap news|'
+    r'live science|science news,|sciencedaily|'
+    r'^\s*\d+\s*$|^[\w\s]{1,15}:?\s*$)',   # very short / numeric only
+    re.IGNORECASE
+)
+
+RSS_FEEDS = {
+    'technology': [
+        'https://feeds.arstechnica.com/arstechnica/index',
+        'https://www.wired.com/feed/rss',
+        'https://hnrss.org/frontpage',
+        'https://www.theverge.com/rss/index.xml',
+        'https://dev.to/feed',
+        'https://thenextweb.com/feed/',
+    ],
+    'science': [
+        'https://www.sciencedaily.com/rss/all.xml',
+        'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
+        'http://export.arxiv.org/rss/cs.AI',
+        'https://phys.org/rss-feed/breaking/',
+    ],
+    'world': [
+        'https://feeds.bbci.co.uk/news/world/rss.xml',
+        'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
+        'https://www.aljazeera.com/xml/rss/all.xml',
+        'https://feeds.npr.org/1004/rss.xml',
+        'https://www.theguardian.com/world/rss',
+    ],
+    'sports':        ['https://feeds.bbci.co.uk/sport/rss.xml',
+                      'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml'],
+    'business':      ['https://feeds.bbci.co.uk/news/business/rss.xml',
+                      'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
+                      'https://www.theguardian.com/business/rss'],
+    'health':        ['https://feeds.bbci.co.uk/news/health/rss.xml',
+                      'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml'],
+    'entertainment': ['https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml',
+                      'https://variety.com/feed/'],
+    'ai':            ['http://export.arxiv.org/rss/cs.AI',
+                      'http://export.arxiv.org/rss/cs.LG',
+                      'https://hnrss.org/frontpage'],
+}
+
+REDDIT_SUBS = [
+    ('technology',    'technology'),
+    ('science',       'science'),
+    ('world',         'worldnews'),
+    ('sports',        'sports'),
+    ('business',      'business'),
+    ('health',        'Health'),
+    ('entertainment', 'movies'),
+    ('ai',            'MachineLearning'),
+    ('ai',            'artificial'),
+    ('technology',    'programming'),
+]
+
+WIKI_TOPICS = [
+    ('science',     ['Physics','Chemistry','Biology','Astronomy','Mathematics','Evolution','Genetics']),
+    ('technology',  ['Artificial intelligence','Machine learning','Computer science','Internet','Robotics']),
+    ('world',       ['Climate change','Democracy','United Nations','Geopolitics','Economics']),
+    ('health',      ['Medicine','Vaccine','Cancer','Nutrition','Mental health','COVID-19']),
+    ('ai',          ['Neural network','Deep learning','Natural language processing','GPT','Transformer model']),
+    ('business',    ['Stock market','Cryptocurrency','Inflation','Supply chain','Startup']),
+    ('entertainment',['Film','Music','Video game','Television','Streaming']),
+    ('sports',      ['Football','Basketball','Olympic Games','Tennis','Cricket']),
+]
+
+HN_TOP = 'https://hacker-news.firebaseio.com/v0/topstories.json'
+HN_ITEM = 'https://hacker-news.firebaseio.com/v0/item/{}.json'
+
+# ── UTILS ────────────────────────────────────────────────────────────────────
+def clean(text, max_chars=800):
+    if not text: return ''
+    text = unescape(str(text))
+    text = re.sub(r'&#?[a-zA-Z0-9]+;', ' ', text)
+    text = re.sub(r'<[^>]+>', ' ', text)
+    text = re.sub(r'http\S+', '', text)
+    text = re.sub(r'[^\w\s.,!?;:\'\-–—]', ' ', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text[:max_chars]
+
+def is_garbage(text):
+    """Return True if text is a website tagline/nav text, not real content."""
+    if len(text) < 40: return True
+    if GARBAGE_PATTERNS.search(text): return True
+    # If it's just a title with no real sentence, skip
+    words = text.split()
+    if len(words) < 8: return True
+    return False
+
+def make_item(text, category, source, extra=None):
+    text = clean(text)
+    if not text or is_garbage(text): return None
+    return {'text': text, 'category': category, 'source': source,
+            'timestamp': datetime.utcnow().isoformat(), **(extra or {})}
+
+def extract_article_text(url, max_chars=600):
+    """Fetch a URL and extract real paragraph text using BeautifulSoup."""
+    if not BS4: return ''
+    try:
+        r = requests.get(url, headers=HEADERS, timeout=8)
+        soup = BeautifulSoup(r.text, 'html.parser')
+        # Remove nav, header, footer, ads
+        for tag in soup(['nav','header','footer','script','style','aside',
+                         'figure','form','button','iframe']):
+            tag.decompose()
+        # Get paragraphs
+        paras = soup.find_all('p')
+        text = ' '.join(p.get_text(' ', strip=True) for p in paras if len(p.get_text()) > 60)
+        return clean(text, max_chars)
+    except Exception:
+        return ''
+
+# ── FETCHERS ─────────────────────────────────────────────────────────────────
+def fetch_rss(category, url):
+    items = []
+    try:
+        feed = feedparser.parse(url)
+        for entry in feed.entries[:12]:
+            title   = entry.get('title', '')
+            summary = entry.get('summary', entry.get('description', ''))
+            # Prefer summary if it has real content (>100 chars)
+            body = summary if len(summary) > 100 else ''
+            text = f"{title}. {body}".strip()
+            item = make_item(text, category, 'rss', {'feed': url.split('/')[2]})
+            if item: items.append(item)
+    except Exception: pass
+    return items
+
+def fetch_reddit(category, subreddit):
+    items = []
+    try:
+        url = f"https://www.reddit.com/r/{subreddit}/top.json?limit=20&t=day"
+        r   = requests.get(url, headers=HEADERS, timeout=10)
+        r.raise_for_status()
+        for post in r.json().get('data',{}).get('children',[]):
+            d = post.get('data',{})
+            title    = d.get('title','')
+            selftext = d.get('selftext','')
+            # Reddit selftext often has real content
+            text = f"{title}. {selftext}" if len(selftext) > 80 else title
+            item = make_item(text, category, 'reddit',
+                           {'subreddit': subreddit, 'score': d.get('score',0)})
+            if item: items.append(item)
+    except Exception: pass
+    return items
+
+def fetch_wikipedia(topic, category):
+    """Fetch real Wikipedia article content — actual knowledge, not taglines."""
+    try:
+        url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{quote_plus(topic)}"
+        r   = requests.get(url, headers=HEADERS, timeout=10)
+        r.raise_for_status()
+        data  = r.json()
+        title = data.get('title','')
+        extract = data.get('extract','')  # Wikipedia gives full intro paragraph
+        if len(extract) < 80: return None
+        text = f"{title}: {extract}"
+        return make_item(text, category, 'wikipedia', {'title': title})
+    except Exception: return None
+
+def fetch_hackernews(n=6):
+    items = []
+    try:
+        ids    = requests.get(HN_TOP, headers=HEADERS, timeout=8).json()[:40]
+        chosen = random.sample(ids, min(n, len(ids)))
+        for sid in chosen:
+            try:
+                story = requests.get(HN_ITEM.format(sid), headers=HEADERS, timeout=5).json()
+                title = story.get('title','')
+                text_body = story.get('text','')
+                text = f"{title}. {text_body}" if len(text_body)>60 else title
+                item = make_item(text, 'technology', 'hackernews',
+                               {'score': story.get('score',0)})
+                if item: items.append(item)
+                time.sleep(0.05)
+            except Exception: continue
+    except Exception: pass
+    return items
+
+# ── MAIN FETCHER ─────────────────────────────────────────────────────────────
+class DataFetcher:
+    def __init__(self):
+        self.total_fetched = 0
+        self.source_counts = {'rss':0,'reddit':0,'wikipedia':0,'hackernews':0}
+        self.recent_items  = deque(maxlen=100)
+        self.log           = deque(maxlen=300)
+        self._wiki_idx     = 0
+
+    def _log(self, msg):
+        ts = datetime.utcnow().strftime('%H:%M:%S')
+        self.log.appendleft(f"[{ts}] {msg}")
+
+    def fetch_round(self):
+        all_items = []
+
+        # 1. RSS — 2 random feeds
+        for _ in range(2):
+            cat = random.choice(list(RSS_FEEDS.keys()))
+            url = random.choice(RSS_FEEDS[cat])
+            items = fetch_rss(cat, url)
+            all_items.extend(items)
+            self.source_counts['rss'] += len(items)
+            self._log(f"📰 RSS [{cat.upper()}] +{len(items)} ← {url.split('/')[2]}")
+
+        # 2. Reddit
+        cat, sub = random.choice(REDDIT_SUBS)
+        items = fetch_reddit(cat, sub)
+        all_items.extend(items)
+        self.source_counts['reddit'] += len(items)
+        self._log(f"🟠 Reddit [r/{sub}] +{len(items)}")
+
+        # 3. Wikipedia — rotate through topics (REAL knowledge content)
+        flat = [(cat, t) for cat, topics in WIKI_TOPICS for t in topics]
+        cat, topic = flat[self._wiki_idx % len(flat)]
+        self._wiki_idx += 1
+        item = fetch_wikipedia(topic, cat)
+        if item:
+            all_items.append(item)
+            self.source_counts['wikipedia'] += 1
+            self._log(f"📖 Wikipedia: {topic}")
+
+        # 4. HackerNews (every other round)
+        if random.random() < 0.5:
+            items = fetch_hackernews(5)
+            all_items.extend(items)
+            self.source_counts['hackernews'] += len(items)
+            self._log(f"💻 HackerNews +{len(items)}")
+
+        for item in all_items:
+            self.recent_items.appendleft(item)
+        self.total_fetched += len(all_items)
+        self._log(f"✅ Round done — +{len(all_items)} | total {self.total_fetched}")
+        return all_items
+
+    def get_stats(self):
+        return {'total_fetched': self.total_fetched,
+                'sources': dict(self.source_counts),
+                'recent_log': list(self.log)[:30]}
+
+    def get_recent_items(self, n=20):
+        return list(self.recent_items)[:n]