Fix _get_hot_topics: count frequency across feeds, sort by hotness (most mentioned first)"

app_v2_entry.py

@@ -1,7 +1,7 @@
 """
 VNEWS v2 Entry Point
-Hashtag: 8 sources (vnexpress, dantri, vietnamnet, bongda, genk, thanhnien, tuoitre, thethaovanhoa)
-Hot topics: from 8 RSS feeds diverse sources
+Hashtag: 8 sources
+Hot topics: SORTED by frequency (most mentioned across feeds = hottest = auto-loaded first)
 """
 import sys, os
 from main import app, HEADERS, BONGDA_HEADERS, fetch_bongda_api
@@ -30,7 +30,6 @@ def _has_kw(topic,title):
     if not words:return True
     return any(w in tt for w in words)
 
-# === 8 SEARCH SOURCES ===
 def _s_vnexpress(topic,limit=8):
     items=[]
     try:
@@ -42,7 +41,6 @@ def _s_vnexpress(topic,limit=8):
                 if _has_kw(topic,t):items.append({'title':t,'url':a['href'],'via':'VnExpress'})
     except:pass
     return items
-
 def _s_dantri(topic,limit=8):
     items=[]
     try:
@@ -55,7 +53,6 @@ def _s_dantri(topic,limit=8):
             if len(items)>=limit:break
     except:pass
     return items
-
 def _s_vietnamnet(topic,limit=6):
     items=[]
     try:
@@ -68,7 +65,6 @@ def _s_vietnamnet(topic,limit=6):
             if len(items)>=limit:break
     except:pass
     return items
-
 def _s_bongda(topic,limit=5):
     items=[]
     try:
@@ -81,7 +77,6 @@ def _s_bongda(topic,limit=5):
             if len(items)>=limit:break
     except:pass
     return items
-
 def _s_genk(topic,limit=5):
     items=[]
     try:
@@ -94,7 +89,6 @@ def _s_genk(topic,limit=5):
             if len(items)>=limit:break
     except:pass
     return items
-
 def _s_thanhnien(topic,limit=6):
     items=[]
     try:
@@ -107,7 +101,6 @@ def _s_thanhnien(topic,limit=6):
             if len(items)>=limit:break
     except:pass
     return items
-
 def _s_tuoitre(topic,limit=6):
     items=[]
     try:
@@ -120,7 +113,6 @@ def _s_tuoitre(topic,limit=6):
             if len(items)>=limit:break
     except:pass
     return items
-
 def _s_thethaovanhoa(topic,limit=5):
     items=[]
     try:
@@ -135,23 +127,12 @@ def _s_thethaovanhoa(topic,limit=5):
     return items
 
 def _search_all(topic, limit=36):
-    """Search 8 sources in parallel, interleave results."""
     results={}
     with ThreadPoolExecutor(8) as ex:
-        futs={
-            ex.submit(_s_vnexpress,topic,8):'vne',
-            ex.submit(_s_dantri,topic,8):'dt',
-            ex.submit(_s_vietnamnet,topic,6):'vnn',
-            ex.submit(_s_bongda,topic,5):'bd',
-            ex.submit(_s_genk,topic,5):'gk',
-            ex.submit(_s_thanhnien,topic,6):'tn',
-            ex.submit(_s_tuoitre,topic,6):'tt',
-            ex.submit(_s_thethaovanhoa,topic,5):'tvh',
-        }
+        futs={ex.submit(_s_vnexpress,topic,8):'vne',ex.submit(_s_dantri,topic,8):'dt',ex.submit(_s_vietnamnet,topic,6):'vnn',ex.submit(_s_bongda,topic,5):'bd',ex.submit(_s_genk,topic,5):'gk',ex.submit(_s_thanhnien,topic,6):'tn',ex.submit(_s_tuoitre,topic,6):'tt',ex.submit(_s_thethaovanhoa,topic,5):'tvh'}
         for f in as_completed(futs,timeout=14):
             try:results[futs[f]]=f.result()
             except:results[futs[f]]=[]
-    # Interleave: take 1 from each source in round-robin
     srcs=list(results.values());interleaved=[];seen=set()
     mx=max((len(s) for s in srcs),default=0)
     for i in range(mx):
@@ -161,40 +142,32 @@ def _search_all(topic, limit=36):
                 if url and url not in seen:seen.add(url);interleaved.append(s[i])
     return interleaved[:limit]
 
-# === ARTICLE SCRAPER ===
+# === ARTICLE ===
 app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/api/article' and 'GET' in getattr(r,'methods',set()))]
 def _scrape_generic(url):
     try:
         r=req.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,allow_redirects=True)
         r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript']):tag.decompose()
-        h1=soup.find('h1');ogt=soup.find('meta',property='og:title')
-        title=(h1.get_text(strip=True) if h1 else '')or(ogt.get('content','') if ogt else '')
+        h1=soup.find('h1');ogt=soup.find('meta',property='og:title');title=(h1.get_text(strip=True) if h1 else '')or(ogt.get('content','') if ogt else '')
         ogd=soup.find('meta',property='og:description');summary=ogd.get('content','') if ogd else ''
         ogi=soup.find('meta',property='og:image');og_img=ogi.get('content','') if ogi else ''
         if og_img and og_img.startswith('//'):og_img='https:'+og_img
         if og_img and 'cdnphoto.dantri' in og_img:og_img='/api/proxy/img?url='+quote(og_img,safe='')
-        selectors=['article','.singular-content','.detail-content','.fck_detail','.content-detail','.knc-content','main','.cms-body','.article__body','.detail__content']
         block=None
-        for sel in selectors:
+        for sel in['article','.singular-content','.detail-content','.fck_detail','.content-detail','.knc-content','main','.cms-body','.article__body']:
             el=soup.select_one(sel)
             if el and len(el.find_all('p'))>=2:block=el;break
         if not block:block=soup.body or soup
         body=[]
         for el in block.find_all(['p','h2','h3','figure','img'],recursive=True):
-            if el.name=='p':
-                t=el.get_text(strip=True)
-                if t and len(t)>30:body.append({'type':'p','text':t})
-            elif el.name in('h2','h3'):
-                t=el.get_text(strip=True)
-                if t:body.append({'type':'heading','text':t})
+            if el.name=='p':t=el.get_text(strip=True);(body.append({'type':'p','text':t}) if t and len(t)>30 else None)
+            elif el.name in('h2','h3'):t=el.get_text(strip=True);(body.append({'type':'heading','text':t}) if t else None)
             elif el.name in('figure','img'):
                 im=el if el.name=='img' else el.find('img')
                 if im:
                     src=im.get('data-src') or im.get('src') or im.get('data-original') or''
-                    if src and'base64' not in src:
-                        if src.startswith('//'):src='https:'+src
-                        body.append({'type':'img','src':src})
+                    if src and'base64' not in src:body.append({'type':'img','src':'https:'+src if src.startswith('//') else src})
         if not body and summary:body=[{'type':'p','text':summary}]
         return{'title':_clean(title),'summary':_clean(summary),'og_image':og_img,'body':body[:50],'source':'generic','url':url}
     except:return None
@@ -212,44 +185,53 @@ def api_article_v2(url:str=Query(...)):
     data=_scrape_generic(url)
     return JSONResponse(data if data else{'error':'Không đọc được','url':url})
 
-# === HOT TOPICS (from 8 diverse RSS feeds) ===
+# === HOT TOPICS — SORTED BY FREQUENCY (most mentioned = hottest = first) ===
 _hot_cache={'t':0,'d':[]}
 def _get_hot_topics():
     now=time.time()
     if _hot_cache['d'] and now-_hot_cache['t']<600:return _hot_cache['d']
-    topics=[];seen=set()
-    feeds=[
-        'https://vnexpress.net/rss/tin-moi-nhat.rss',
-        'https://dantri.com.vn/rss/home.rss',
-        'https://vietnamnet.vn/rss/tin-moi-nhat.rss',
-        'https://thanhnien.vn/rss/home.rss',
-        'https://tuoitre.vn/rss/tin-moi-nhat.rss',
-        'https://genk.vn/rss',
-        'https://vnexpress.net/rss/the-thao.rss',
-        'https://thethaovanhoa.vn/rss/tin-nong.rss',
-    ]
+    
+    # Count how many times each phrase appears across ALL feeds
+    freq={};display={}
+    feeds=['https://vnexpress.net/rss/tin-moi-nhat.rss','https://dantri.com.vn/rss/home.rss','https://vietnamnet.vn/rss/tin-moi-nhat.rss','https://thanhnien.vn/rss/home.rss','https://tuoitre.vn/rss/tin-moi-nhat.rss','https://genk.vn/rss','https://vnexpress.net/rss/the-thao.rss','https://thethaovanhoa.vn/rss/tin-nong.rss']
+    
     for feed_url in feeds:
         try:
             r=req.get(feed_url,headers={'User-Agent':'Mozilla/5.0'},timeout=6);r.encoding='utf-8';soup=BeautifulSoup(r.text,'xml')
-            for item in soup.find_all('item')[:8]:
+            for item in soup.find_all('item')[:12]:
                 title=_clean(item.find('title').get_text() if item.find('title') else '')
                 if not title:continue
                 title=re.sub(r'\s*[-|].*$','',title)
                 words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP]
                 if len(words)<2:continue
-                phrase=' '.join(words[:4])
-                if len(phrase)<8 or len(phrase)>45:continue
-                key=phrase.lower()
-                if key in seen:continue
-                is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen)
-                if is_dup:continue
-                seen.add(key);topics.append({'label':'#'+re.sub(r'\s+','',phrase.title()),'topic':phrase})
-                if len(topics)>=20:break
+                # Generate phrases and count
+                for n in(3,4,2):
+                    for i in range(max(0,len(words)-n+1)):
+                        phrase=' '.join(words[i:i+n])
+                        if len(phrase)<8 or len(phrase)>45:continue
+                        key=phrase.lower()
+                        freq[key]=freq.get(key,0)+1
+                        display[key]=phrase
         except:continue
+    
+    # Sort by frequency (most mentioned = hottest)
+    ranked=sorted(freq.items(),key=lambda x:x[1],reverse=True)
+    
+    topics=[];seen=set()
+    for key,count in ranked:
+        # Skip duplicates (overlapping phrases)
+        is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen)
+        if is_dup:continue
+        seen.add(key)
+        phrase=display[key]
+        topics.append({'label':'#'+re.sub(r'\s+','',phrase.title()),'topic':phrase,'count':count})
         if len(topics)>=20:break
-    for kw in['World Cup 2026','Kinh tế Việt Nam','Bóng đá châu Âu','Công nghệ AI','Giá vàng','Thời tiết','An ninh mạng','Xe điện']:
+    
+    # Add fallbacks
+    for kw in['World Cup 2026','Kinh tế Việt Nam','Bóng đá châu Âu','Công nghệ AI','Giá vàng','Thời tiết','An ninh mạng']:
         if len(topics)>=24:break
-        if not any(kw.lower() in s for s in seen):topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw})
+        if not any(kw.lower() in s for s in seen):topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw,'count':0})
+    
     _hot_cache.update({'t':now,'d':topics[:24]});return topics[:24]
 
 @app.get('/api/hot_topics')
@@ -262,7 +244,7 @@ async def serve_index():
 @app.get('/api/hashtag/sources')
 def _ht(topic:str=Query(...),page:int=Query(default=0)):
     items=_search_all(topic,36)
-    per_page=6;start=page*per_page;end=start+per_page
+    per_page=8;start=page*per_page;end=start+per_page
     return JSONResponse({'sources':items[start:end],'topic':topic,'page':page,'has_more':end<len(items),'total':len(items)})
 @app.get('/api/categories')
 def _cat():return JSONResponse([])