Hashtag search + hot topics: add bongda & bongdaplus (Jina); Jina article fallback

app_v2_entry.py

@@ -71,7 +71,29 @@ def _s_vietnamnet(topic,limit=6):
             if len(items)>=limit:break
     except:pass
     return items
-def _s_bongda(topic,limit=5):
+def _s_bongda(topic,limit=6):
+    """bongda.com.vn via Jina homepage (site is JS-protected); keyword-filtered."""
+    items=[]
+    try:
+        from main import scrape_bongda_jina
+        for a in scrape_bongda_jina(40):
+            if _has_kw(topic,a.get('title','')):
+                items.append({'title':a['title'],'url':a['link'],'via':'Bóng Đá'})
+            if len(items)>=limit:break
+    except:pass
+    return items
+def _s_bongdaplus(topic,limit=6):
+    """bongdaplus.vn via Jina homepage (JS SPA); keyword-filtered."""
+    items=[]
+    try:
+        from main import scrape_bongdaplus_jina
+        for a in scrape_bongdaplus_jina(40):
+            if _has_kw(topic,a.get('title','')):
+                items.append({'title':a['title'],'url':a['link'],'via':'Bóng Đá+'})
+            if len(items)>=limit:break
+    except:pass
+    return items
+def _s_bongda_OLD(topic,limit=5):
     items=[]
     try:
         r=req.get(f"https://bongda.com.vn/tim-kiem.html?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
@@ -133,9 +155,9 @@ def _s_thethaovanhoa(topic,limit=5):
     return items
 def _search_all(topic,limit=36):
     results={}
-    with ThreadPoolExecutor(8) as ex:
-        futs={ex.submit(_s_vnexpress,topic,8):'vne',ex.submit(_s_dantri,topic,8):'dt',ex.submit(_s_vietnamnet,topic,6):'vnn',ex.submit(_s_bongda,topic,5):'bd',ex.submit(_s_genk,topic,5):'gk',ex.submit(_s_thanhnien,topic,6):'tn',ex.submit(_s_tuoitre,topic,6):'tt',ex.submit(_s_thethaovanhoa,topic,5):'tvh'}
-        for f in as_completed(futs,timeout=14):
+    with ThreadPoolExecutor(10) as ex:
+        futs={ex.submit(_s_vnexpress,topic,8):'vne',ex.submit(_s_dantri,topic,8):'dt',ex.submit(_s_vietnamnet,topic,6):'vnn',ex.submit(_s_bongda,topic,6):'bd',ex.submit(_s_bongdaplus,topic,6):'bdp',ex.submit(_s_genk,topic,5):'gk',ex.submit(_s_thanhnien,topic,6):'tn',ex.submit(_s_tuoitre,topic,6):'tt',ex.submit(_s_thethaovanhoa,topic,5):'tvh'}
+        for f in as_completed(futs,timeout=16):
             try:results[futs[f]]=f.result()
             except:results[futs[f]]=[]
     srcs=list(results.values());out=[];seen=set();mx=max((len(s) for s in srcs),default=0)
@@ -145,9 +167,40 @@ def _search_all(topic,limit=36):
     return out[:limit]
 # Override article endpoint
 app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/api/article' and 'GET' in getattr(r,'methods',set()))]
+def _jina_article(url):
+    """Read an article via r.jina.ai (for Cloudflare/SPA sites: vatvostudio, bongdaplus)."""
+    try:
+        r=req.get("https://r.jina.ai/"+url,headers={'Accept':'text/markdown','X-Return-Format':'markdown','User-Agent':'Mozilla/5.0'},timeout=35)
+        if r.status_code!=200 or not r.text:return None
+        lines=[x.rstrip() for x in r.text.splitlines()]
+        title='';og_img='';summary='';body=[]
+        for ln in lines[:40]:
+            if ln.startswith('Title:') and not title:title=ln.replace('Title:','',1).strip()
+        # first content image
+        for ln in lines:
+            mi=re.search(r'!\[[^\]]*\]\((https?://[^)]+)\)',ln)
+            if mi and 'wp-content' in mi.group(1) or (mi and 'media' in mi.group(1)):og_img=mi.group(1);break
+        intxt=False
+        for ln in lines:
+            if ln.startswith('Markdown Content:'):intxt=True;continue
+            if not intxt:continue
+            t=re.sub(r'!\[[^\]]*\]\([^)]+\)','',ln)            # drop images
+            t=re.sub(r'\[([^\]]+)\]\([^)]+\)',r'\1',t)         # unwrap links
+            t=re.sub(r'[#>*_`]+','',t).strip()
+            if len(t)>=40:body.append({'type':'p','text':t})
+            if len(body)>=60:break
+        if not title and body:title=body[0]['text'][:90]
+        if not body:return None
+        return {'title':_clean(title),'summary':_clean(body[0]['text'][:200]) if body else '','og_image':og_img,'body':body,'source':'jina','url':url}
+    except:return None
 def _scrape_generic(url):
     try:
-        r=req.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,allow_redirects=True);r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
+        r=req.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,allow_redirects=True);r.encoding='utf-8'
+        # JS SPA / Cloudflare-protected sites -> read via Jina
+        if r.status_code>=400 or 'bongdaplus.vn' in url or 'vatvostudio.vn' in url or 'Just a moment' in r.text[:600]:
+            jr=_jina_article(url)
+            if jr and jr.get('body'):return jr
+        soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript']):tag.decompose()
         h1=soup.find('h1');ogt=soup.find('meta',property='og:title');title=(h1.get_text(strip=True) if h1 else '')or(ogt.get('content','') if ogt else '')
         ogd=soup.find('meta',property='og:description');summary=ogd.get('content','') if ogd else ''
@@ -166,8 +219,12 @@ def _scrape_generic(url):
                 im=el if el.name=='img' else el.find('img')
                 if im:src=im.get('data-src') or im.get('src') or'';(body.append({'type':'img','src':'https:'+src if src.startswith('//') else src}) if src and'base64' not in src else None)
         if not body and summary:body=[{'type':'p','text':summary}]
+        if not body:
+            jr=_jina_article(url)
+            if jr and jr.get('body'):return jr
         return{'title':_clean(title),'summary':_clean(summary),'og_image':og_img,'body':body[:50],'source':'generic','url':url}
-    except:return None
+    except:
+        return _jina_article(url)
 @app.get('/api/article')
 def api_article_v2(url:str=Query(...)):
     from main import scrape_vne_article,scrape_bbc_article,scrape_dantri_article,scrape_genk_article,scrape_ttvh_article
@@ -198,6 +255,23 @@ def _get_hot_topics():
                         phrase=' '.join(words[i:i+n])
                         if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase
         except:continue
+    # Add bongda.com.vn + bongdaplus.vn (JS sites, no RSS) via Jina-scraped homepage titles
+    def _ingest_titles(titles):
+        for title in titles:
+            title=_clean(title)
+            if not title:continue
+            title=re.sub(r'\s*[-|].*$','',title)
+            words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP]
+            if len(words)<2:continue
+            for n in(3,4,2):
+                for i in range(max(0,len(words)-n+1)):
+                    phrase=' '.join(words[i:i+n])
+                    if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase
+    try:
+        from main import scrape_bongda_jina,scrape_bongdaplus_jina
+        _ingest_titles([a.get('title','') for a in scrape_bongda_jina(20)])
+        _ingest_titles([a.get('title','') for a in scrape_bongdaplus_jina(20)])
+    except:pass
     ranked=sorted(freq.items(),key=lambda x:x[1],reverse=True);topics=[];seen=set()
     for key,count in ranked:
         is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen)