app_run.py

"""Wrapper: hashtag via Google News with pagination, strict relevance, load more."""
from app_final import *
from app_final import app, f6, f5, rt, PATCH_INJECT, UNIFIED_INJECT_FIXED, HIGHLIGHT_FULL_OVERRIDE, EXTRA_WALL_FIX
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi import Query, Request
import requests as req
from urllib.parse import quote
from bs4 import BeautifulSoup
import re, html as html_lib

def _clean(s):return re.sub(r"\s+"," ",html_lib.unescape(str(s or ""))).strip()

def _follow_redirect(url):
    try:
        r=req.head(url,allow_redirects=True,timeout=10,headers={'User-Agent':'Mozilla/5.0'})
        return r.url
    except:
        try:r=req.get(url,allow_redirects=True,timeout=10,headers={'User-Agent':'Mozilla/5.0'},stream=True);u=r.url;r.close();return u
        except:return url

def _scrape_any_article(url):
    if 'news.google.com' in url or 'google.com/rss' in url:url=_follow_redirect(url)
    try:
        r=req.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9,en;q=0.8'},timeout=15,allow_redirects=True)
        r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
        for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe']):tag.decompose()
        h1=soup.find('h1');ogt=soup.find('meta',property='og:title')
        title=(h1.get_text(' ',strip=True) if h1 else '') or (ogt.get('content','') if ogt else '') or (soup.title.get_text(strip=True) if soup.title else '')
        ogd=soup.find('meta',property='og:description') or soup.find('meta',attrs={'name':'description'})
        summary=ogd.get('content','') if ogd else ''
        ogi=soup.find('meta',property='og:image') or soup.find('meta',attrs={'name':'twitter:image'})
        og_image=ogi.get('content','') if ogi else ''
        if og_image and og_image.startswith('//'):og_image='https:'+og_image
        selectors=['article','main','.article-content','.detail-content','.singular-content','.fck_detail','.content-detail','.entry-content','.story-body','.knc-content','.cms-body']
        block=None
        for sel in selectors:
            el=soup.select_one(sel)
            if el and len(el.find_all('p'))>=2:block=el;break
        if not block:
            best=None;best_score=0
            for el in soup.find_all(['article','main','section','div']):
                ps=el.find_all('p');score=len(ps)*100+sum(len(p.get_text())for p in ps[:10])
                if score>best_score:best=el;best_score=score
            block=best or soup.body or soup
        body=[]
        for el in block.find_all(['p','h2','h3','figure','img'],recursive=True):
            if el.name=='p':
                t=_clean(el.get_text(' ',strip=True))
                if len(t)>30:body.append({'type':'p','text':t})
            elif el.name in ('h2','h3'):
                t=_clean(el.get_text(' ',strip=True))
                if t:body.append({'type':'heading','text':t})
            elif el.name in ('figure','img'):
                im=el if el.name=='img' else el.find('img')
                if im:
                    src=im.get('data-src') or im.get('data-original') or im.get('src') or ''
                    if src and 'base64' not in src:
                        if src.startswith('//'):src='https:'+src
                        body.append({'type':'img','src':src})
        if not body and summary:body=[{'type':'p','text':summary}]
        return {'title':_clean(title),'summary':_clean(summary),'og_image':og_image,'body':body[:50],'source':'generic','url':url}
    except:return None

def _google_news_search_all(topic, limit=30):
    """Get ALL results from Google News RSS for a topic — no filtering here, filter in endpoint."""
    items=[]
    try:
        url='https://news.google.com/rss/search?q='+quote(topic)+'&hl=vi&gl=VN&ceid=VN:vi'
        r=req.get(url,headers={'User-Agent':'Mozilla/5.0'},timeout=10);r.encoding='utf-8'
        soup=BeautifulSoup(r.text,'xml')
        for it in soup.find_all('item')[:limit]:
            title=_clean(it.find('title').get_text(' ',strip=True) if it.find('title') else '')
            link=_clean(it.find('link').get_text(strip=True) if it.find('link') else '')
            src=_clean(it.find('source').get_text(' ',strip=True) if it.find('source') else '')
            pub=_clean(it.find('pubDate').get_text(strip=True) if it.find('pubDate') else '')
            if not title or not link:continue
            items.append({'title':title,'url':link,'via':src,'snippet':'','pubDate':pub})
    except:pass
    return items

def _filter_relevant(items, topic):
    """Strict filter: topic keywords MUST appear in title."""
    topic_lower=topic.lower()
    topic_words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',topic_lower) if len(w)>2]
    filtered=[]
    for s in items:
        title_lower=s.get('title','').lower()
        # Whole phrase match OR majority of words match
        if topic_lower in title_lower:
            filtered.append(s);continue
        if topic_words:
            match=sum(1 for w in topic_words if w in title_lower)
            if match>=len(topic_words)*0.6:
                filtered.append(s)
    return filtered

# Override endpoints
app.router.routes=[r for r in app.router.routes if not (
    (getattr(r,'path',None)=='/api/hashtag/sources' and 'GET' in getattr(r,'methods',set())) or
    (getattr(r,'path',None)=='/api/article' and 'GET' in getattr(r,'methods',set())) or
    (getattr(r,'path',None)=='/' and 'GET' in getattr(r,'methods',set()))
)]

@app.get('/api/article')
def _article_universal(url:str=Query(...)):
    data=_scrape_any_article(url)
    if data and data.get('body'):return JSONResponse(data)
    from main import scrape_vne_article,scrape_bbc_article,scrape_dantri_article,scrape_genk_article,scrape_ttvh_article
    if 'vnexpress.net' in url:d=scrape_vne_article(url)
    elif 'bbc.com' in url:d=scrape_bbc_article(url)
    elif 'dantri.com.vn' in url:d=scrape_dantri_article(url)
    elif 'genk.vn' in url:d=scrape_genk_article(url)
    elif 'thethaovanhoa.vn' in url:d=scrape_ttvh_article(url)
    else:d=None
    if d and d.get('body'):return JSONResponse(d)
    return JSONResponse({'error':'Không đọc được bài viết','url':url})

@app.get('/api/hashtag/sources')
def _hashtag_paged(topic:str=Query(...),page:int=Query(default=0)):
    """Google News search with pagination. page=0 returns first 6, page=1 returns next 6, etc."""
    all_items=_google_news_search_all(topic,30)
    filtered=_filter_relevant(all_items,topic)
    # If strict filter too harsh, fallback to all
    if len(filtered)<3:filtered=all_items
    per_page=6;start=page*per_page;end=start+per_page
    page_items=filtered[start:end]
    has_more=end<len(filtered)
    return JSONResponse({'sources':page_items,'topic':topic,'page':page,'has_more':has_more,'total':len(filtered)})

FAST_HASHTAG_JS = r'''
<style>
.hashtag-loading{display:flex;align-items:center;gap:8px;padding:12px;color:#888;font-size:12px}
.hashtag-spinner{width:16px;height:16px;border:2px solid #333;border-top-color:#5cb87a;border-radius:50%;animation:ht-spin .8s linear infinite}
@keyframes ht-spin{to{transform:rotate(360deg)}}
.hashtag-load-more{width:100%;margin-top:8px;background:#222;border:1px solid #333;color:#ccc;padding:9px;border-radius:10px;font-size:12px;cursor:pointer}.hashtag-load-more:active{opacity:.7}
</style>
<script>
(function(){
function esc(s){return String(s||'').replace(/[&<>"']/g,function(m){return{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[m]});}
var _htPage=0,_htTopic='',_htImgIdx=0;

window.readArticle=async function(url){
  showView('view-article');var el=document.getElementById('view-article');el.innerHTML='<div class="loading">Đang tải...</div>';
  try{var r=await fetch('/api/article?url='+encodeURIComponent(url));var data=await r.json();
    if(data&&!data.error&&data.body&&data.body.length){window._currentArticle={url:url,data:data};var h='<button class="back-btn" onclick="switchCat(\'home\')">← Quay lại</button><div class="article-view"><h1 class="article-title">'+esc(data.title)+'</h1>';if(data.summary)h+='<div class="article-summary">'+esc(data.summary)+'</div>';var seen={};data.body.forEach(function(b){if(b.type==='p')h+='<p class="article-p">'+b.text+'</p>';else if(b.type==='img'&&b.src&&!seen[b.src]){seen[b.src]=1;h+='<img class="article-img" src="'+esc(b.src)+'" onerror="this.style.display=\'none\'">';}else if(b.type==='heading')h+='<h2 class="article-h2">'+esc(b.text)+'</h2>';});h+='<div class="article-actions"><button class="primary" onclick="doRewriteArticle(this)">🤖 Rewrite AI đăng tường</button><button onclick="doShare(\''+esc(data.title)+'\',\''+esc(url)+'\',\''+esc(data.og_image||'')+'\')">📤</button><button onclick="window.open(\''+esc(url)+'\',\'_blank\')">🔗 Gốc</button></div><div class="article-ai-ask"><h3 style="font-size:14px;color:#5cb87a">🤖 Hỏi AI</h3><textarea id="article-ai-question" placeholder="Hỏi..."></textarea><button onclick="askArticleAI()">Hỏi</button><div id="article-ai-answer" class="article-ai-answer"></div></div></div>';el.innerHTML=h;window.scrollTo(0,0);return;}}catch(e){}
  el.innerHTML='<button class="back-btn" onclick="switchCat(\'home\')">← Quay lại</button><div class="loading"><p>Không đọc được.</p><a href="'+esc(url)+'" target="_blank" style="color:#5cb87a">Mở gốc →</a></div>';
};
window.doRewriteArticle=async function(btn){var url=(window._currentArticle&&window._currentArticle.url)||'';if(!url){alert('Không có URL');return;}var ctx=document.querySelector('.article-view')?.innerText?.slice(0,14000)||'';btn.disabled=true;btn.textContent='Đang rewrite...';try{var r=await fetch('/api/rewrite_share',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url:url,context:ctx})});var j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');alert('Đã đăng Tường AI!');}catch(e){alert(e.message);}finally{btn.disabled=false;btn.textContent='🤖 Rewrite AI đăng tường';}};
window.askArticleAI=async function(){var q=document.getElementById('article-ai-question')?.value.trim();if(!q)return alert('Nhập câu hỏi');var a=document.getElementById('article-ai-answer');a.textContent='Đang hỏi...';var url=(window._currentArticle&&window._currentArticle.url)||'';var ctx=document.querySelector('.article-view')?.innerText?.slice(0,12000)||'';try{var r=await fetch('/api/article/ask',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url:url,question:q,context:ctx})});var j=await r.json();a.textContent=j.answer||'Không trả lời được';}catch(e){a.textContent='Lỗi: '+e.message}};

function renderSources(sources,append){
  var list=document.getElementById('hashtag-src-list');if(!list)return;
  var h='';
  sources.forEach(function(s){
    var idx=_htImgIdx++;
    h+='<div class="hashtag-src-item" onclick="readArticle(\''+esc(s.url||'')+'\')">';
    h+='<div class="hashtag-src-img" id="ht-img-'+idx+'"></div>';
    h+='<div class="hashtag-src-text"><div class="hashtag-src-title">'+esc(s.title)+'</div><div class="hashtag-src-via">'+esc(s.via||'')+(s.pubDate?' · '+esc(s.pubDate.split(',')[0]||''):'')+'</div></div>';
    h+='</div>';
    // Lazy load image
    setTimeout(function(){fetch('/api/article?url='+encodeURIComponent(s.url)).then(function(r){return r.json()}).then(function(d){if(d&&(d.og_image||d.img)){var el=document.getElementById('ht-img-'+idx);if(el)el.innerHTML='<img src="'+esc(d.og_image||d.img)+'" onerror="this.style.display=\'none\'" loading="lazy">';}}).catch(function(){});},idx*500);
  });
  if(append)list.insertAdjacentHTML('beforeend',h);else list.innerHTML=h;
}

window.showHashtagSources=async function(topic){
  _htTopic=topic;_htPage=0;_htImgIdx=0;
  var home=document.getElementById('view-home');if(!home)return;
  document.getElementById('hashtag-sources-box')?.remove();
  var box=document.createElement('div');box.id='hashtag-sources-box';box.className='hashtag-sources';
  box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div class="hashtag-loading"><div class="hashtag-spinner"></div>Đang tìm bài viết mới nhất...</div>';
  var compose=home.querySelector('.ai-compose');
  if(compose)compose.after(box);else home.prepend(box);
  box.scrollIntoView({behavior:'smooth',block:'start'});
  try{
    var r=await fetch('/api/hashtag/sources?topic='+encodeURIComponent(topic)+'&page=0');
    var j=await r.json();var sources=j.sources||[];
    if(!sources.length){box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div style="color:#888;font-size:12px;padding:8px">Không tìm được bài viết liên quan</div>';return;}
    var h='<h3>🔍 '+esc(topic)+' <span style="font-size:10px;color:#888">('+j.total+' bài mới nhất từ Google News)</span></h3>';
    h+='<div id="hashtag-src-list"></div>';
    h+='<button class="hashtag-rewrite-btn" onclick="rewriteHashtagTopic(\''+esc(topic)+'\')">🤖 Rewrite AI tổng hợp & đăng tường</button>';
    if(j.has_more)h+='<button class="hashtag-load-more" id="ht-load-more" onclick="loadMoreSources()">Tải thêm bài viết ▼</button>';
    box.innerHTML=h;
    renderSources(sources,false);
  }catch(e){box.innerHTML='<h3>🔍 '+esc(topic)+'</h3><div style="color:#e74c3c;font-size:12px;padding:8px">Lỗi: '+esc(e.message)+'</div>';}
};

window.loadMoreSources=async function(){
  _htPage++;var btn=document.getElementById('ht-load-more');
  if(btn){btn.textContent='Đang tải...';btn.disabled=true;}
  try{
    var r=await fetch('/api/hashtag/sources?topic='+encodeURIComponent(_htTopic)+'&page='+_htPage);
    var j=await r.json();var sources=j.sources||[];
    renderSources(sources,true);
    if(!j.has_more&&btn)btn.remove();
    else if(btn){btn.textContent='Tải thêm bài viết ▼';btn.disabled=false;}
  }catch(e){if(btn){btn.textContent='Lỗi, thử lại';btn.disabled=false;}}
};

window.rewriteHashtagTopic=async function(topic){var btn=event?.target;if(btn){btn.disabled=true;btn.textContent='Đang tổng hợp...';}try{var r=await fetch('/api/topic_post',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({topic:topic})});var j=await r.json();if(!r.ok||j.error)throw new Error(j.error||'Lỗi');if(btn)btn.textContent='✅ Đã đăng!';setTimeout(function(){document.getElementById('hashtag-sources-box')?.remove();},2000);}catch(e){if(btn){btn.disabled=false;btn.textContent='❌ '+e.message;}}};
window.createTopicPost=function(){var inp=document.getElementById('ai-topic-input');var topic=(inp&&inp.value||'').trim();if(!topic){alert('Nhập chủ đề');return;}showHashtagSources(topic);if(inp)inp.value='';};
window.createTopicPostFinal5=function(){var inp=document.getElementById('ai-topic-input-final5')||document.getElementById('ai-topic-input');var topic=(inp&&inp.value||'').trim();if(!topic){alert('Nhập chủ đề');return;}showHashtagSources(topic);if(inp)inp.value='';};
})();
</script>
'''

@app.get('/')
async def _index_run():
    html=f5.f4.f3.f2.f1._load_index_html()
    body=''
    body+=getattr(rt.old,'PATCH_INJECT','')
    body+=f5.f4.f3.f2.f1.FINAL_INJECT+f5.f4.f3.FINAL3_INJECT+f5.f4.FINAL4_INJECT+f5.FINAL5_INJECT
    body+=getattr(f6,'FINAL6_INJECT','')
    body+=getattr(f6,'FINAL6_FAST_HOME_INJECT','')
    body+=getattr(f6,'FINAL6E_INJECT','')
    body+=PATCH_INJECT
    body+=UNIFIED_INJECT_FIXED
    body+=HIGHLIGHT_FULL_OVERRIDE
    body+=EXTRA_WALL_FIX
    body+=FAST_HASHTAG_JS
    return HTMLResponse(html.replace('</body>',body+'\n</body>') if '</body>' in html else html+body)