Add AI-specific scrapers: GenK AI (ai.chn), Tinhte, VatvoStudio, KhoaHoc.tv for AI Thế Giới / AI Việt Nam topics - sorted by newest first

app_v2_entry.py

@@ -219,6 +219,165 @@ def _s_date(el):
     if dt: return _clean(dt.get_text(strip=True))[:20]
     return ''
 
+# ===== AI-specific scrapers (GenK AI, Tinhte, VatvoStudio, KhoaHoc.tv) =====
+def _s_genk_ai(limit=10):
+    """Scrape GenK AI section (ai.chn) for the latest AI articles."""
+    items=[]
+    try:
+        r=req.get("https://genk.vn/ai.chn", headers={'User-Agent':'Mozilla/5.0'}, timeout=10)
+        if r.status_code!=200: return items
+        r.encoding="utf-8"; soup=BeautifulSoup(r.text,'lxml')
+        for a in soup.find_all('a', href=True)[:limit*3]:
+            href=a.get('href','')
+            if not href.endswith('.chn') or href=='/ai.chn': continue
+            if href.startswith('/'): href='https://genk.vn'+href
+            t=_clean(a.get('title','') or a.get_text(strip=True))
+            if not t or len(t)<15: continue
+            # Extract date from article page
+            date_str = ''
+            img_src = ''
+            # Get date from parent block
+            date_el = a.parent
+            for _ in range(5):
+                if date_el is None: break
+                dt=date_el.find('time') or date_el.select_one('.date, .time, span.date, .meta-date')
+                if dt:
+                    date_str=_clean(dt.get_text(strip=True))[:20]
+                    break
+                date_el=date_el.parent
+            # Get image
+            container=a.parent
+            for _ in range(6):
+                if container is None: break
+                for img in container.find_all('img'):
+                    s=img.get('data-src','') or img.get('src','')
+                    if s and 'mediacdn' in s and 'avatar' not in s: img_src=s; break
+                if img_src: break; container=container.parent
+            if not img_src:
+                try:
+                    og_r=req.get(href, headers={'User-Agent':'Mozilla/5.0'}, timeout=5)
+                    og_r.encoding='utf-8'
+                    og_s=BeautifulSoup(og_r.text,'lxml')
+                    og_tag=og_s.find('meta', property='og:image')
+                    if og_tag: img_src=og_tag.get('content','')
+                    meta_date=og_s.find('meta', property='article:published_time')
+                    if meta_date and meta_date.get('content'): date_str=meta_date.get('content')[:19]
+                except: pass
+            items.append({'title':t[:200], 'url':href, 'via':'GenK AI', 'img':img_src, 'date': date_str})
+            if len(items)>=limit: break
+    except: pass
+    return items
+
+def _s_tinhte_ai(limit=10):
+    """Scrape Tinhte.vn for AI articles (tag: tri-tue-nhan-tao, ai)."""
+    items=[]
+    seen_urls=set()
+    for tag in ['tri-tue-nhan-tao', 'ai']:
+        try:
+            r=req.get(f"https://tinhte.vn/tags/{tag}/", headers={'User-Agent':'Mozilla/5.0'}, timeout=10)
+            if r.status_code!=200: continue
+            r.encoding='utf-8'; soup=BeautifulSoup(r.text,'lxml')
+            for a in soup.find_all('a', href=True)[:limit*2]:
+                href=a.get('href','')
+                if 'tinhte.vn' not in href and not href.startswith('/threads/'): continue
+                if href.startswith('/'): href='https://tinhte.vn'+href
+                if href in seen_urls: continue
+                t=_clean(a.get('title','') or a.get_text(strip=True))
+                if not t or len(t)<15: continue
+                seen_urls.add(href)
+                img_src=''
+                img=a.find('img') or (a.parent.find('img') if a.parent else None)
+                if img: img_src=img.get('data-src','') or img.get('src','') or ''
+                date_str=_s_date(a.parent or a)
+                items.append({'title':t[:200], 'url':href, 'via':'Tinhte', 'img':img_src, 'date':date_str})
+                if len(items)>=limit: break
+        except: pass
+    return items[:limit]
+
+def _s_vatvostudio_ai(limit=10):
+    """Scrape VatvoStudio for AI articles."""
+    items=[]
+    try:
+        r=req.get("https://vatvostudio.vn/", headers={'User-Agent':'Mozilla/5.0'}, timeout=10)
+        if r.status_code!=200: return items
+        r.encoding='utf-8'; soup=BeautifulSoup(r.text,'lxml')
+        for a in soup.find_all('a', href=True)[:limit*4]:
+            href=a.get('href','')
+            if 'vatvostudio.vn' not in href and not href.startswith('/'): continue
+            t=_clean(a.get('title','') or a.get_text(strip=True))
+            if not t or len(t)<15: continue
+            ai_kw = ['ai', 'trí tuệ', 'machine learning', 'deep learning', 'chatgpt', 'gemini', 'claude', 'llm', 'openai', 'neural', 'robot', 'thông minh']
+            t_lower = t.lower()
+            if not any(kw in t_lower for kw in ai_kw): continue
+            if not href.startswith('http'): href='https://vatvostudio.vn'+href
+            img_src=''
+            img=a.find('img') or (a.parent.find('img') if a.parent else None)
+            if img: img_src=img.get('data-src','') or img.get('src','') or ''
+            date_str=_s_date(a.parent or a)
+            items.append({'title':t[:200], 'url':href, 'via':'VatvoStudio', 'img':img_src, 'date':date_str})
+            if len(items)>=limit: break
+    except: pass
+    return items
+
+def _s_khoahoc_tv_ai(limit=5):
+    """Scrape khoahoc.tv for AI articles."""
+    items=[]
+    try:
+        r=req.get("https://khoahoc.tv/tin-tuc/cong-nghe", headers={'User-Agent':'Mozilla/5.0'}, timeout=10)
+        if r.status_code!=200: return items
+        r.encoding='utf-8'; soup=BeautifulSoup(r.text,'lxml')
+        for a in soup.find_all('a', href=True)[:limit*3]:
+            href=a.get('href','')
+            t=_clean(a.get('title','') or a.get_text(strip=True))
+            if not t or len(t)<15: continue
+            ai_kw = ['ai', 'trí tuệ', 'machine learning', 'chatgpt', 'gemini', 'claude', 'openai', 'robot']
+            if not any(kw in t.lower() for kw in ai_kw): continue
+            if href.startswith('/'): href='https://khoahoc.tv'+href
+            img_src=''
+            img=a.find('img') or (a.parent.find('img') if a.parent else None)
+            if img: img_src=img.get('data-src','') or img.get('src','') or ''
+            date_str=_s_date(a.parent or a)
+            items.append({'title':t[:200], 'url':href, 'via':'KhoaHoc.tv', 'img':img_src, 'date':date_str})
+            if len(items)>=limit: break
+    except: pass
+    return items
+
+def _search_ai(topic, limit=36):
+    """Search AI-specific sources, sorted by newest first."""
+    results={}
+    with ThreadPoolExecutor(4) as ex:
+        futs={
+            ex.submit(_s_genk_ai, 12): 'genk',
+            ex.submit(_s_tinhte_ai, 10): 'tinhte',
+            ex.submit(_s_vatvostudio_ai, 10): 'vatvo',
+            ex.submit(_s_khoahoc_tv_ai, 6): 'khtv',
+        }
+        for f in as_completed(futs, timeout=18):
+            try: results[futs[f]] = f.result()
+            except: results[futs[f]] = []
+    # Deduplicate
+    out=[]; seen=set()
+    for key in ['genk', 'tinhte', 'vatvo', 'khtv']:
+        for s in results.get(key, []):
+            if s.get('url') and s['url'] not in seen:
+                seen.add(s['url']); out.append(s)
+    # Sort by date (newest first)
+    def _date_sort_key(s):
+        d = s.get('date', '')
+        if not d: return '0000'
+        # "yyyy-mm-ddTHH:MM:SS" or "yyyy-mm-dd HH:MM:SS"
+        m = re.search(r'(\d{4})-(\d{2})-(\d{2})', d)
+        if m: return m.group(1)+m.group(2)+m.group(3)
+        # "dd/mm/yyyy"
+        m = re.search(r'(\d{2})/(\d{2})/(\d{4})', d)
+        if m: return m.group(3)+m.group(2)+m.group(1)
+        # "X phút trước", "X giờ trước" - very recent
+        if 'phút' in d or 'giờ' in d or 'Vừa' in d: return '9999'
+        if 'Hôm qua' in d: return '9998'
+        return '0000'
+    out.sort(key=_date_sort_key, reverse=True)
+    return out[:limit]
+
 def _s_vnexpress(topic,limit=8):
     items=[]
     try:
@@ -301,7 +460,7 @@ def _s_tuoitre(topic,limit=6):
     try:
         r=req.get(f"https://tuoitre.vn/tim-kiem.htm?keywords={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
         for a in soup.select('h3 a[href], .box-title-text a')[:limit*2]:
-            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
+            t=_clean(a.get_text(strip=True));href=a.get('href','')
             if t and len(t)>15 and _has_kw(topic,t):
                 if not href.startswith('http'):href='https://tuoitre.vn'+href
                 items.append({'title':t,'url':href,'via':'Tuổi Trẻ','date':_s_date(a.parent or a)})
@@ -538,7 +697,13 @@ async def serve_index():
     return HTMLResponse('<h1>VNEWS</h1>')
 @app.get('/api/hashtag/sources')
 def _ht(topic:str=Query(...),page:int=Query(default=0)):
-    items=_search_all(topic,36);per_page=8;start=page*per_page;end=start+per_page
+    ai_kw = ['ai việt nam', 'ai thế giới', 'công nghệ ai', 'trí tuệ nhân tạo', 'ai', 'trí tuệ']
+    is_ai = any(kw in topic.lower() for kw in ai_kw)
+    if is_ai:
+        items=_search_ai(topic,36)
+    else:
+        items=_search_all(topic,36)
+    per_page=8;start=page*per_page;end=start+per_page
     return JSONResponse({'sources':items[start:end],'topic':topic,'page':page,'has_more':end<len(items),'total':len(items)})
 @app.get('/api/categories')
 def _cat():return JSONResponse([])
@@ -1603,7 +1768,7 @@ async def _auto_rewrite_one(topic, slot_label, used_urls=None, post_index=0):
     if not items or post_index >= len(items):
         return False
     
-    # Get article at post_index (0,1,2 for multiple posts)
+    # Get article at post_index (0,1,2 for multiple articles)
     item = items[post_index]  # post_index allows multiple articles per topic
     url = item.get('url', '')
     title = item.get('title', topic)
@@ -1904,4 +2069,4 @@ def shorts_rss():
     
     return {"shorts": shorts, "count": len(shorts)}
 
-app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static')
+app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static')