Fix: use VnExpress/Dantri/Vietnamnet direct search for real URLs, not Google News"

app_run.py

@@ -1,4 +1,4 @@
-"""Wrapper: relevant hashtag with pagination + ALL articles in-app."""
+"""Wrapper: real article URLs from VN news search, universal scrape, pagination."""
 from app_final import *
 from app_final import app, f6, f5, rt, PATCH_INJECT, UNIFIED_INJECT_FIXED, HIGHLIGHT_FULL_OVERRIDE, EXTRA_WALL_FIX
 from fastapi.responses import HTMLResponse, JSONResponse
@@ -7,29 +7,80 @@ import requests as req
 from urllib.parse import quote
 from bs4 import BeautifulSoup
 import re, html as html_lib
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 def _clean(s):return re.sub(r"\s+"," ",html_lib.unescape(str(s or ""))).strip()
+UA={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'}
 
-def _follow_redirect(url):
+def _vne_search(topic, limit=10):
+    """Search VnExpress — returns real URLs."""
+    items=[]
     try:
-        r=req.head(url,allow_redirects=True,timeout=10,headers={'User-Agent':'Mozilla/5.0'})
-        return r.url
-    except:
-        try:r=req.get(url,allow_redirects=True,timeout=10,headers={'User-Agent':'Mozilla/5.0'},stream=True);u=r.url;r.close();return u
-        except:return url
+        r=req.get(f'https://timkiem.vnexpress.net/?q={quote(topic)}',headers=UA,timeout=8);r.encoding='utf-8'
+        soup=BeautifulSoup(r.text,'lxml')
+        for a in soup.select('article.item-news h2 a, article.item-news h3 a, h2.title-news a, h3.title-news a')[:limit]:
+            title=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
+            if title and href and href.startswith('http'):items.append({'title':title,'url':href,'via':'VnExpress'})
+    except:pass
+    return items
+
+def _dantri_search(topic, limit=8):
+    """Search Dantri — returns real URLs."""
+    items=[]
+    try:
+        r=req.get(f'https://dantri.com.vn/tim-kiem/{quote(topic)}.htm',headers=UA,timeout=8);r.encoding='utf-8'
+        soup=BeautifulSoup(r.text,'lxml')
+        for a in soup.select('h3.article-title a, h2.article-title a, .article-item a[href]')[:limit]:
+            title=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
+            if not href.startswith('http'):href='https://dantri.com.vn'+href
+            if title and len(title)>15 and href.endswith('.htm'):items.append({'title':title,'url':href,'via':'Dân trí'})
+    except:pass
+    return items
+
+def _vietnamnet_search(topic, limit=8):
+    """Search Vietnamnet — returns real URLs."""
+    items=[]
+    try:
+        r=req.get(f'https://vietnamnet.vn/tim-kiem?q={quote(topic)}',headers=UA,timeout=8);r.encoding='utf-8'
+        soup=BeautifulSoup(r.text,'lxml')
+        for a in soup.select('h3 a[href], .verticalPost__main a[href], .horizontalPost__main a[href]')[:limit]:
+            title=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
+            if not href.startswith('http'):href='https://vietnamnet.vn'+href
+            if title and len(title)>15:items.append({'title':title,'url':href,'via':'Vietnamnet'})
+    except:pass
+    return items
+
+def _multi_search(topic, limit=20):
+    """Search multiple VN news sources in parallel for real URLs."""
+    all_items=[];seen=set()
+    with ThreadPoolExecutor(3) as ex:
+        futs=[ex.submit(_vne_search,topic,10),ex.submit(_dantri_search,topic,8),ex.submit(_vietnamnet_search,topic,8)]
+        for f in as_completed(futs,timeout=12):
+            try:
+                for item in f.result():
+                    if item['url'] not in seen:seen.add(item['url']);all_items.append(item)
+            except:pass
+    # Filter relevance
+    topic_lower=topic.lower()
+    topic_words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',topic_lower) if len(w)>1]
+    relevant=[]
+    for item in all_items:
+        title_lower=item['title'].lower()
+        if topic_lower in title_lower:relevant.append(item);continue
+        match=sum(1 for w in topic_words if w in title_lower)
+        if match>=max(1,len(topic_words)//2):relevant.append(item)
+    return relevant[:limit]
 
 def _scrape_any_article(url):
-    if 'news.google.com' in url or 'google.com/rss' in url:url=_follow_redirect(url)
     try:
-        r=req.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,allow_redirects=True)
-        r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
+        r=req.get(url,headers=UA,timeout=15,allow_redirects=True);r.encoding='utf-8'
+        soup=BeautifulSoup(r.text,'lxml')
         for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe']):tag.decompose()
         h1=soup.find('h1');ogt=soup.find('meta',property='og:title')
         title=(h1.get_text(' ',strip=True) if h1 else '') or (ogt.get('content','') if ogt else '') or (soup.title.get_text(strip=True) if soup.title else '')
         ogd=soup.find('meta',property='og:description') or soup.find('meta',attrs={'name':'description'})
         summary=ogd.get('content','') if ogd else ''
-        ogi=soup.find('meta',property='og:image') or soup.find('meta',attrs={'name':'twitter:image'})
-        og_image=ogi.get('content','') if ogi else ''
+        ogi=soup.find('meta',property='og:image');og_image=ogi.get('content','') if ogi else ''
         if og_image and og_image.startswith('//'):og_image='https:'+og_image
         selectors=['article','main','.article-content','.detail-content','.singular-content','.fck_detail','.entry-content','.story-body','.knc-content','.cms-body']
         block=None
@@ -37,19 +88,15 @@ def _scrape_any_article(url):
             el=soup.select_one(sel)
             if el and len(el.find_all('p'))>=2:block=el;break
         if not block:
-            best=None;best_score=0
+            best=None;bs=0
             for el in soup.find_all(['article','main','section','div']):
-                ps=el.find_all('p');score=len(ps)*100+sum(len(p.get_text())for p in ps[:10])
-                if score>best_score:best=el;best_score=score
+                ps=el.find_all('p');sc=len(ps)*100+sum(len(p.get_text())for p in ps[:10])
+                if sc>bs:best=el;bs=sc
             block=best or soup.body or soup
         body=[]
         for el in block.find_all(['p','h2','h3','figure','img'],recursive=True):
-            if el.name=='p':
-                t=_clean(el.get_text(' ',strip=True))
-                if len(t)>30:body.append({'type':'p','text':t})
-            elif el.name in('h2','h3'):
-                t=_clean(el.get_text(' ',strip=True))
-                if t:body.append({'type':'heading','text':t})
+            if el.name=='p':t=_clean(el.get_text(' ',strip=True));(body.append({'type':'p','text':t}) if len(t)>30 else None)
+            elif el.name in('h2','h3'):t=_clean(el.get_text(' ',strip=True));(body.append({'type':'heading','text':t}) if t else None)
             elif el.name in('figure','img'):
                 im=el if el.name=='img' else el.find('img')
                 if im:
@@ -61,52 +108,6 @@ def _scrape_any_article(url):
         return {'title':_clean(title),'summary':_clean(summary),'og_image':og_image,'body':body[:50],'source':'generic','url':url}
     except:return None
 
-def _google_news_search(topic, limit=20):
-    """Search Google News RSS — returns newest first, strict relevance."""
-    items=[]
-    try:
-        # Use exact topic as search query for better relevance
-        url='https://news.google.com/rss/search?q='+quote('"'+topic+'"')+'&hl=vi&gl=VN&ceid=VN:vi'
-        r=req.get(url,headers={'User-Agent':'Mozilla/5.0'},timeout=10);r.encoding='utf-8'
-        soup=BeautifulSoup(r.text,'xml')
-        for it in soup.find_all('item')[:limit*3]:
-            title=_clean(it.find('title').get_text(' ',strip=True) if it.find('title') else '')
-            link=_clean(it.find('link').get_text(strip=True) if it.find('link') else '')
-            src=_clean(it.find('source').get_text(' ',strip=True) if it.find('source') else '')
-            pub=_clean(it.find('pubDate').get_text(strip=True) if it.find('pubDate') else '')
-            if not title or not link:continue
-            # Strict relevance: topic phrase or majority of words must be in title
-            topic_lower=topic.lower();title_lower=title.lower()
-            topic_words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',topic_lower) if len(w)>1]
-            if topic_lower in title_lower:
-                pass  # exact match — always include
-            else:
-                match_count=sum(1 for w in topic_words if w in title_lower)
-                if len(topic_words)>0 and match_count<max(1,len(topic_words)//2):continue
-            items.append({'title':title,'url':link,'via':src,'snippet':'','pub':pub})
-            if len(items)>=limit:break
-    except:pass
-    # Also try without quotes for broader results if too few
-    if len(items)<5:
-        try:
-            url2='https://news.google.com/rss/search?q='+quote(topic+' Việt Nam')+'&hl=vi&gl=VN&ceid=VN:vi'
-            r2=req.get(url2,headers={'User-Agent':'Mozilla/5.0'},timeout=8);r2.encoding='utf-8'
-            soup2=BeautifulSoup(r2.text,'xml')
-            seen_urls={s['url'] for s in items}
-            for it in soup2.find_all('item')[:limit]:
-                title=_clean(it.find('title').get_text(' ',strip=True) if it.find('title') else '')
-                link=_clean(it.find('link').get_text(strip=True) if it.find('link') else '')
-                src=_clean(it.find('source').get_text(' ',strip=True) if it.find('source') else '')
-                if not title or not link or link in seen_urls:continue
-                topic_lower=topic.lower();title_lower=title.lower()
-                topic_words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',topic_lower) if len(w)>1]
-                match_count=sum(1 for w in topic_words if w in title_lower)
-                if match_count==0:continue
-                seen_urls.add(link);items.append({'title':title,'url':link,'via':src,'snippet':''})
-                if len(items)>=limit:break
-        except:pass
-    return items
-
 # Override endpoints
 app.router.routes=[r for r in app.router.routes if not (
     (getattr(r,'path',None)=='/api/hashtag/sources' and 'GET' in getattr(r,'methods',set())) or
@@ -115,7 +116,7 @@ app.router.routes=[r for r in app.router.routes if not (
 )]
 
 @app.get('/api/article')
-def _article_universal(url:str=Query(...)):
+def _article(url:str=Query(...)):
     data=_scrape_any_article(url)
     if data and data.get('body'):return JSONResponse(data)
     from main import scrape_vne_article,scrape_bbc_article,scrape_dantri_article,scrape_genk_article,scrape_ttvh_article
@@ -129,24 +130,20 @@ def _article_universal(url:str=Query(...)):
     return JSONResponse({'error':'Không đọc được bài viết','url':url})
 
 @app.get('/api/hashtag/sources')
-def _hashtag_paginated(topic:str=Query(...),offset:int=Query(default=0),limit:int=Query(default=6)):
-    """Return relevant sources with pagination. Newest first."""
-    all_sources=_google_news_search(topic,20)
-    # Dedupe
-    seen=set();unique=[]
-    for s in all_sources:
-        if s['url'] not in seen:seen.add(s['url']);unique.append(s)
-    total=len(unique)
-    page=unique[offset:offset+limit]
+def _hashtag(topic:str=Query(...),offset:int=Query(default=0),limit:int=Query(default=6)):
+    """Return relevant sources from VN news search with real URLs. Supports pagination."""
+    all_sources=_multi_search(topic,20)
+    total=len(all_sources)
+    page=all_sources[offset:offset+limit]
     has_more=offset+limit<total
-    return JSONResponse({'sources':page,'topic':topic,'total':total,'offset':offset,'has_more':has_more})
+    return JSONResponse({'sources':[{'title':s['title'],'url':s['url'],'via':s['via']} for s in page],'topic':topic,'total':total,'offset':offset,'has_more':has_more})
 
 FAST_HASHTAG_JS = r'''
 <style>
 .hashtag-loading{display:flex;align-items:center;gap:8px;padding:12px;color:#888;font-size:12px}
 .hashtag-spinner{width:16px;height:16px;border:2px solid #333;border-top-color:#5cb87a;border-radius:50%;animation:ht-spin .8s linear infinite}
 @keyframes ht-spin{to{transform:rotate(360deg)}}
-.hashtag-load-more{width:100%;margin-top:8px;background:#222;border:1px solid #333;color:#ccc;padding:9px;border-radius:10px;font-size:12px;cursor:pointer;text-align:center}.hashtag-load-more:active{opacity:.7}
+.hashtag-load-more{width:100%;margin-top:8px;background:#222;border:1px solid #333;color:#ccc;padding:9px;border-radius:10px;font-size:12px;cursor:pointer;text-align:center}
 </style>
 <script>
 (function(){
@@ -170,57 +167,36 @@ window.readArticle=async function(url){
 window.doRewriteArticle=async function(btn){var url=(window._currentArticle&&window._currentArticle.url)||'';if(!url){alert('Không có URL');return;}var ctx=document.querySelector('.article-view')?.innerText?.slice(0,14000)||'';btn.disabled=true;btn.textContent='Đang rewrite...';try{var r=await fetch('/api/rewrite_share',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url:url,context:ctx})});var j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');alert('Đã đăng Tường AI!');}catch(e){alert(e.message);}finally{btn.disabled=false;btn.textContent='🤖 Rewrite AI đăng tường';}};
 window.askArticleAI=async function(){var q=document.getElementById('article-ai-question')?.value.trim();if(!q)return alert('Nhập câu hỏi');var a=document.getElementById('article-ai-answer');a.textContent='Đang hỏi...';var url=(window._currentArticle&&window._currentArticle.url)||'';var ctx=document.querySelector('.article-view')?.innerText?.slice(0,12000)||'';try{var r=await fetch('/api/article/ask',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url:url,question:q,context:ctx})});var j=await r.json();a.textContent=j.answer||'Không trả lời được';}catch(e){a.textContent='Lỗi: '+e.message}};
 
-// === HASHTAG SOURCES with pagination ===
 function renderSources(sources,append){
-  var list=document.getElementById('hashtag-src-list');if(!list)return;
-  var h='';
-  sources.forEach(function(s){
-    var idx=_htImgIdx++;
+  var list=document.getElementById('hashtag-src-list');if(!list)return;var h='';
+  sources.forEach(function(s){var idx=_htImgIdx++;
     h+='<div class="hashtag-src-item" onclick="readArticle(\''+esc(s.url||'')+'\')">';
     h+='<div class="hashtag-src-img" id="ht-img-'+idx+'"></div>';
-    h+='<div class="hashtag-src-text"><div class="hashtag-src-title">'+esc(s.title)+'</div><div class="hashtag-src-via">'+esc(s.via||'')+'</div></div>';
-    h+='</div>';
-    // Lazy load image
-    if(s.url)fetch('/api/article?url='+encodeURIComponent(s.url)).then(function(r){return r.json()}).then(function(d){if(d&&(d.og_image||d.img)){var el=document.getElementById('ht-img-'+idx);if(el)el.innerHTML='<img src="'+esc(d.og_image||d.img)+'" onerror="this.style.display=\'none\'" loading="lazy">';}}).catch(function(){});
+    h+='<div class="hashtag-src-text"><div class="hashtag-src-title">'+esc(s.title)+'</div><div class="hashtag-src-via">'+esc(s.via||'')+'</div></div></div>';
+    fetch('/api/article?url='+encodeURIComponent(s.url)).then(function(r){return r.json()}).then(function(d){if(d&&(d.og_image||d.img)){var el=document.getElementById('ht-img-'+idx);if(el)el.innerHTML='<img src="'+esc(d.og_image||d.img)+'" onerror="this.style.display=\'none\'" loading="lazy">';}}).catch(function(){});
   });
   if(append)list.insertAdjacentHTML('beforeend',h);else list.innerHTML=h;
 }
-
 window.showHashtagSources=async function(topic){
   _htTopic=topic;_htOffset=0;_htImgIdx=0;
   var home=document.getElementById('view-home');if(!home)return;
   document.getElementById('hashtag-sources-box')?.remove();
   var box=document.createElement('div');box.id='hashtag-sources-box';box.className='hashtag-sources';
   box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div class="hashtag-loading"><div class="hashtag-spinner"></div>Đang tìm bài viết mới nhất...</div>';
-  var compose=home.querySelector('.ai-compose');
-  if(compose)compose.after(box);else home.prepend(box);
+  var compose=home.querySelector('.ai-compose');if(compose)compose.after(box);else home.prepend(box);
   box.scrollIntoView({behavior:'smooth',block:'start'});
   try{
     var r=await fetch('/api/hashtag/sources?topic='+encodeURIComponent(topic)+'&offset=0&limit=6');
     var j=await r.json();var sources=j.sources||[];
-    if(!sources.length){box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div style="color:#888;font-size:12px;padding:8px">Không tìm được bài viết liên quan</div>';return;}
+    if(!sources.length){box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div style="color:#888;font-size:12px;padding:8px">Không tìm được bài viết</div>';return;}
     _htOffset=sources.length;
-    var h='<h3>🔍 '+esc(topic)+' <span style="font-size:10px;color:#888">('+j.total+' bài)</span></h3>';
-    h+='<div id="hashtag-src-list"></div>';
+    var h='<h3>🔍 '+esc(topic)+' <span style="font-size:10px;color:#888">('+j.total+' bài)</span></h3><div id="hashtag-src-list"></div>';
     if(j.has_more)h+='<button class="hashtag-load-more" id="ht-load-more" onclick="loadMoreHashtag()">Tải thêm bài viết...</button>';
     h+='<button class="hashtag-rewrite-btn" onclick="rewriteHashtagTopic(\''+esc(topic)+'\')">🤖 Rewrite AI tổng hợp & đăng tường</button>';
-    box.innerHTML=h;
-    renderSources(sources,false);
-  }catch(e){box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div style="color:#e74c3c;font-size:12px;padding:8px">Lỗi: '+esc(e.message)+'</div>';}
-};
-
-window.loadMoreHashtag=async function(){
-  var btn=document.getElementById('ht-load-more');if(btn){btn.textContent='Đang tải...';btn.disabled=true;}
-  try{
-    var r=await fetch('/api/hashtag/sources?topic='+encodeURIComponent(_htTopic)+'&offset='+_htOffset+'&limit=6');
-    var j=await r.json();var sources=j.sources||[];
-    _htOffset+=sources.length;
-    renderSources(sources,true);
-    if(!j.has_more&&btn)btn.remove();
-    else if(btn){btn.textContent='Tải thêm bài viết...';btn.disabled=false;}
-  }catch(e){if(btn){btn.textContent='Lỗi, thử lại';btn.disabled=false;}}
+    box.innerHTML=h;renderSources(sources,false);
+  }catch(e){box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div style="color:#e74c3c;font-size:12px">Lỗi: '+esc(e.message)+'</div>';}
 };
-
+window.loadMoreHashtag=async function(){var btn=document.getElementById('ht-load-more');if(btn){btn.textContent='Đang tải...';btn.disabled=true;}try{var r=await fetch('/api/hashtag/sources?topic='+encodeURIComponent(_htTopic)+'&offset='+_htOffset+'&limit=6');var j=await r.json();_htOffset+=j.sources.length;renderSources(j.sources,true);if(!j.has_more&&btn)btn.remove();else if(btn){btn.textContent='Tải thêm...';btn.disabled=false;}}catch(e){if(btn){btn.textContent='Lỗi';btn.disabled=false;}}};
 window.rewriteHashtagTopic=async function(topic){var btn=event?.target;if(btn){btn.disabled=true;btn.textContent='Đang tổng hợp...';}try{var r=await fetch('/api/topic_post',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({topic:topic})});var j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');if(btn)btn.textContent='✅ Đã đăng!';setTimeout(function(){document.getElementById('hashtag-sources-box')?.remove();},2000);}catch(e){if(btn){btn.disabled=false;btn.textContent='❌ '+e.message;}}};
 window.createTopicPost=function(){var inp=document.getElementById('ai-topic-input');var topic=(inp&&inp.value||'').trim();if(!topic){alert('Nhập chủ đề');return;}showHashtagSources(topic);if(inp)inp.value='';};
 window.createTopicPostFinal5=function(){var inp=document.getElementById('ai-topic-input-final5')||document.getElementById('ai-topic-input');var topic=(inp&&inp.value||'').trim();if(!topic){alert('Nhập chủ đề');return;}showHashtagSources(topic);if(inp)inp.value='';};