app_v2_entry.py

"""VNEWS v2 Entry Point - with friendly highlights"""
import sys, os, traceback
from main import app, HEADERS, BONGDA_HEADERS, fetch_bongda_api, HL_LEAGUES
from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
from fastapi.staticfiles import StaticFiles
from starlette.routing import Mount
from fastapi import Query, Request
import requests as req
from urllib.parse import quote
from bs4 import BeautifulSoup
import re, html as html_lib, json, threading, time
from concurrent.futures import ThreadPoolExecutor, as_completed

# Fix friendly path: correct URL is xemlaibongda.top/giai-khac/friendly
HL_LEAGUES['friendly'] = {"path": "giai-khac/friendly", "name": "Giao hữu", "emoji": "🤝"}

STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/' and hasattr(r,'methods') and 'GET' in getattr(r,'methods',set()))]
app.routes[:]=[r for r in app.routes if not isinstance(r, Mount)]
app.router.routes=[r for r in app.router.routes if not isinstance(r, Mount)]
def _clean(s):return re.sub(r"\s+"," ",html_lib.unescape(str(s or""))).strip()
_STOP=set('và của các những một được trong với cho tại sau trước khi không người việt nam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì này đã để'.split())
def _has_kw(topic,title):
    tl=topic.lower();tt=(title or'').lower()
    if tl in tt:return True
    words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',tl) if len(w)>2 and w not in _STOP]
    if not words:return True
    return any(w in tt for w in words)
def _s_vnexpress(topic,limit=8):
    items=[];
    try:
        r=req.get(f"https://timkiem.vnexpress.net/?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for art in soup.select('article.item-news')[:limit]:
            a=art.select_one('h2 a, h3 a')
            if a and a.get('href'):t=_clean(a.get('title','') or a.get_text(strip=True));items.append({'title':t,'url':a['href'],'via':'VnExpress'}) if _has_kw(topic,t) else None
    except:pass
    return items
def _s_dantri(topic,limit=8):
    items=[];
    try:
        r=req.get(f"https://dantri.com.vn/tim-kiem/{quote(topic)}.htm",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .article-title a[href]')[:limit*2]:
            t=_clean(a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://dantri.com.vn'+href
                items.append({'title':t,'url':href,'via':'Dân Trí'})
            if len(items)>=limit:break
    except:pass
    return items
def _s_vietnamnet(topic,limit=6):
    items=[];
    try:
        r=req.get(f"https://vietnamnet.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .vnn-title a')[:limit*2]:
            t=_clean(a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://vietnamnet.vn'+href
                items.append({'title':t,'url':href,'via':'VietNamNet'})
            if len(items)>=limit:break
    except:pass
    return items
def _s_bongda(topic,limit=5):
    items=[];
    try:
        r=req.get(f"https://bongda.com.vn/tim-kiem.html?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .title a[href]')[:limit*2]:
            t=_clean(a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://bongda.com.vn'+href
                items.append({'title':t,'url':href,'via':'Bóng Đá'})
            if len(items)>=limit:break
    except:pass
    return items
def _s_genk(topic,limit=5):
    items=[];
    try:
        r=req.get(f"https://genk.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('a[href$=".chn"]')[:limit*3]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if href.startswith('/'):href='https://genk.vn'+href
                items.append({'title':t,'url':href,'via':'GenK'})
            if len(items)>=limit:break
    except:pass
    return items
def _s_thanhnien(topic,limit=6):
    items=[];
    try:
        r=req.get(f"https://thanhnien.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .box-title a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://thanhnien.vn'+href
                items.append({'title':t,'url':href,'via':'Thanh Niên'})
            if len(items)>=limit:break
    except:pass
    return items
def _s_tuoitre(topic,limit=6):
    items=[];
    try:
        r=req.get(f"https://tuoitre.vn/tim-kiem.htm?keywords={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .box-title-text a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://tuoitre.vn'+href
                items.append({'title':t,'url':href,'via':'Tuổi Trẻ'})
            if len(items)>=limit:break
    except:pass
    return items
def _s_thethaovanhoa(topic,limit=5):
    items=[];
    try:
        r=req.get(f"https://thethaovanhoa.vn/tim-kiem.htm?keyword={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .title a[href]')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://thethaovanhoa.vn'+href
                items.append({'title':t,'url':href,'via':'TT&VH'})
            if len(items)>=limit:break
    except:pass
    return items
def _search_all(topic,limit=36):
    results={}
    with ThreadPoolExecutor(8) as ex:
        futs={ex.submit(_s_vnexpress,topic,8):'vne',ex.submit(_s_dantri,topic,8):'dt',ex.submit(_s_vietnamnet,topic,6):'vnn',ex.submit(_s_bongda,topic,5):'bd',ex.submit(_s_genk,topic,5):'gk',ex.submit(_s_thanhnien,topic,6):'tn',ex.submit(_s_tuoitre,topic,6):'tt',ex.submit(_s_thethaovanhoa,topic,5):'tvh'}
        for f in as_completed(futs,timeout=14):
            try:results[futs[f]]=f.result()
            except:results[futs[f]]=[]
    srcs=list(results.values());out=[];seen=set();mx=max((len(s) for s in srcs),default=0)
    for i in range(mx):
        for s in srcs:
            if i<len(s) and s[i].get('url') and s[i]['url'] not in seen:seen.add(s[i]['url']);out.append(s[i])
    return out[:limit]

def _search_multi_topics(topics, limit=48):
    """Search across multiple topics in parallel, merge and deduplicate results.
    Returns merged list with _matched_topics annotation per article."""
    topic_list = [t.strip() for t in topics if t.strip()]
    if not topic_list:
        return [], []
    if len(topic_list) == 1:
        return _search_all(topic_list[0], limit), topic_list

    # Run _search_all for each topic in parallel
    merged = []
    seen_urls = set()
    with ThreadPoolExecutor(min(len(topic_list), 6)) as ex:
        futs = {ex.submit(_search_all, t, 20): t for t in topic_list}
        for f in as_completed(futs, timeout=18):
            try:
                topic_results = f.result()
                for item in topic_results:
                    url = item.get('url', '')
                    if url and url not in seen_urls:
                        seen_urls.add(url)
                        item['_matched_topics'] = [futs[f]]
                        merged.append(item)
                    elif url:
                        # Already seen — annotate additional matched topic
                        for existing in merged:
                            if existing.get('url') == url:
                                mt = existing.get('_matched_topics', [])
                                if futs[f] not in mt:
                                    mt.append(futs[f])
                                    existing['_matched_topics'] = mt
                                break
            except:
                pass

    # Sort: articles matching more topics first, then by original order
    try:
        merged.sort(key=lambda x: (-len(x.get('_matched_topics', [])), merged.index(x) if x in merged else 999))
    except Exception:
        pass  # If sort fails, keep original order
    return merged[:limit], topic_list

app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/api/article' and 'GET' in getattr(r,'methods',set()))]
def _scrape_generic(url):
    try:
        r=req.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36','Accept-Language':'vi-VN,vi;q=0.9'},timeout=15,allow_redirects=True);r.encoding='utf-8';soup=BeautifulSoup(r.text,'lxml')
        for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript']):tag.decompose()
        h1=soup.find('h1');ogt=soup.find('meta',property='og:title');title=(h1.get_text(strip=True) if h1 else '')or(ogt.get('content','') if ogt else '')
        ogd=soup.find('meta',property='og:description');summary=ogd.get('content','') if ogd else ''
        ogi=soup.find('meta',property='og:image');og_img=ogi.get('content','') if ogi else ''
        if og_img and og_img.startswith('//'):og_img='https:'+og_img
        block=None
        for sel in['article','.singular-content','.detail-content','.fck_detail','.content-detail','.knc-content','main','.cms-body','.article__body']:
            el=soup.select_one(sel)
            if el and len(el.find_all('p'))>=2:block=el;break
        if not block:block=soup.body or soup
        body=[]
        for el in block.find_all(['p','h2','h3','figure','img'],recursive=True):
            if el.name=='p':t=el.get_text(strip=True);(body.append({'type':'p','text':t}) if t and len(t)>30 else None)
            elif el.name in('h2','h3'):t=el.get_text(strip=True);(body.append({'type':'heading','text':t}) if t else None)
            elif el.name in('figure','img'):
                im=el if el.name=='img' else el.find('img')
                if im:src=im.get('data-src') or im.get('src') or'';(body.append({'type':'img','src':'https:'+src if src.startswith('//') else src}) if src and'base64' not in src else None)
        if not body and summary:body=[{'type':'p','text':summary}]
        return{'title':_clean(title),'summary':_clean(summary),'og_image':og_img,'body':body[:50],'source':'generic','url':url}
    except:return None
@app.get('/api/article')
def api_article_v2(url:str=Query(...)):
    from main import scrape_vne_article,scrape_bbc_article,scrape_dantri_article,scrape_genk_article,scrape_ttvh_article
    if 'vnexpress.net' in url:data=scrape_vne_article(url)
    elif 'bbc.com' in url:data=scrape_bbc_article(url)
    elif 'dantri.com.vn' in url:data=scrape_dantri_article(url)
    elif 'genk.vn' in url:data=scrape_genk_article(url)
    elif 'thethaovanhoa.vn' in url:data=scrape_ttvh_article(url)
    else:data=_scrape_generic(url)
    if data and data.get('body'):return JSONResponse(data)
    data=_scrape_generic(url);return JSONResponse(data if data else{'error':'Không đọc được','url':url})
_hot_cache={'t':0,'d':[]}
def _get_hot_topics():
    now=time.time()
    if _hot_cache['d'] and now-_hot_cache['t']<600:return _hot_cache['d']
    freq={};display={}
    feeds=['https://vnexpress.net/rss/tin-moi-nhat.rss','https://dantri.com.vn/rss/home.rss','https://vietnamnet.vn/rss/tin-moi-nhat.rss','https://thanhnien.vn/rss/home.rss','https://tuoitre.vn/rss/tin-moi-nhat.rss','https://genk.vn/rss','https://vnexpress.net/rss/the-thao.rss','https://thethaovanhoa.vn/rss/tin-nong.rss']
    for feed_url in feeds:
        try:
            r=req.get(feed_url,headers={'User-Agent':'Mozilla/5.0'},timeout=6);r.encoding='utf-8';soup=BeautifulSoup(r.text,'xml')
            for item in soup.find_all('item')[:12]:
                title=_clean(item.find('title').get_text() if item.find('title') else '')
                if not title:continue
                title=re.sub(r'\s*[-|].*$','',title);words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP]
                if len(words)<2:continue
                for n in(3,4,2):
                    for i in range(max(0,len(words)-n+1)):
                        phrase=' '.join(words[i:i+n])
                        if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase
        except:continue
    ranked=sorted(freq.items(),key=lambda x:x[1],reverse=True);topics=[];seen=set()
    for key,count in ranked:
        is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen)
        if is_dup:continue
        seen.add(key);topics.append({'label':'#'+re.sub(r'\s+','',display[key].title()),'topic':display[key],'count':count})
        if len(topics)>=20:break
    for kw in['World Cup 2026','Kinh tế Việt Nam','Bóng đá châu Âu','Công nghệ AI','Giá vàng','Thời tiết']:
        if len(topics)>=24:break
        if not any(kw.lower() in s for s in seen):topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw,'count':0})
    _hot_cache.update({'t':now,'d':topics[:24]});return topics[:24]
@app.get('/api/hot_topics')
def api_hot_topics():return JSONResponse({'topics':_get_hot_topics()})
@app.get('/')
async def serve_index():
    p=os.path.join(STATIC_DIR,'index_v2.html')
    if os.path.exists(p):return FileResponse(p,media_type='text/html')
    return HTMLResponse('<h1>VNEWS</h1>')
@app.get('/api/hashtag/sources')
def _ht(topic:str=Query(...),page:int=Query(default=0),extra_topics:str=Query(default='')):
    try:
        # Build list of topics to search: primary + extra from other hashtags
        all_topics = [topic.strip()] if topic.strip() else []
        if extra_topics:
            for t in extra_topics.split(','):
                t = t.strip()
                if t and t not in all_topics:
                    all_topics.append(t)

        if len(all_topics) == 1:
            items = _search_all(all_topics[0], 48)
            search_label = all_topics[0]
        else:
            items, searched_topics = _search_multi_topics(all_topics, 48)
            search_label = ', '.join(all_topics) if len(all_topics) <= 3 else f'{len(all_topics)} hashtags'

        per_page=8;start=page*per_page;end=start+per_page
        return JSONResponse({
            'sources':items[start:end],
            'topic':search_label,
            'page':page,
            'has_more':end<len(items),
            'total':len(items),
            'all_topics':all_topics,
            'multi':len(all_topics)>1
        })
    except Exception as e:
        # Return JSON error instead of HTML 500 page
        return JSONResponse({'error': str(e), 'trace': traceback.format_exc()[-500:]}, status_code=500)
@app.get('/api/categories')
def _cat():return JSONResponse([])
@app.get('/api/storage_status')
def _st():return JSONResponse({'persistent':os.path.isdir('/data') and os.access('/data',os.W_OK)})
@app.get('/s')
async def _sh(url:str='',title:str='',img:str=''):return HTMLResponse(f'<!DOCTYPE html><html><head><meta property="og:title" content="{_clean(title)}"><meta property="og:image" content="{_clean(img)}"><meta http-equiv="refresh" content="0;url={_clean(url) or "/"}"></head><body></body></html>')
DATA_DIR='/data' if os.path.isdir('/data') else os.path.join(os.path.dirname(os.path.abspath(__file__)),'data')
os.makedirs(DATA_DIR,exist_ok=True);IF=os.path.join(DATA_DIR,'interactions_v2.json');CF=os.path.join(DATA_DIR,'comments_v2.json')
_il=threading.Lock();_cl=threading.Lock()
def _lj(p):
    try:
        if os.path.exists(p):return json.load(open(p,'r',encoding='utf-8'))
    except:pass
    return{}
def _sj(p,d):
    try:open(p+'.tmp','w',encoding='utf-8').write(json.dumps(d,ensure_ascii=False));os.replace(p+'.tmp',p)
    except:pass
@app.post('/api/v2/interact')
async def _int(request:Request):
    b=await request.json();v=str(b.get('id','')).strip();t=str(b.get('type','')).strip()
    if not v or t not in('view','like'):return JSONResponse({'error':'x'},status_code=400)
    with _il:db=_lj(IF);db.setdefault(v,{'views':0,'likes':0,'comments':0});db[v][t+'s']+=1;_sj(IF,db);return JSONResponse(db[v])
@app.get('/api/v2/interactions')
def _gi(id:str=Query(...)):
    with _il:return JSONResponse(_lj(IF).get(id.strip(),{'views':0,'likes':0,'comments':0}))
@app.get('/api/v2/comments')
def _gc(id:str=Query(...)):
    with _cl:return JSONResponse({'comments':_lj(CF).get(id.strip(),[])})
@app.post('/api/v2/comment')
async def _pc(request:Request):
    b=await request.json();v=str(b.get('id','')).strip();tx=str(b.get('text','')).strip()[:500]
    if not v or not tx:return JSONResponse({'error':'x'},status_code=400)
    c={'text':tx,'time':time.strftime('%H:%M %d/%m',time.localtime()),'ts':int(time.time())}
    with _cl:db=_lj(CF);db.setdefault(v,[]);db[v].append(c);db[v]=db[v][-200:];_sj(CF,db);cms=db[v]
    with _il:idb=_lj(IF);idb.setdefault(v,{'views':0,'likes':0,'comments':0});idb[v]['comments']=len(cms);_sj(IF,idb)
    return JSONResponse({'comments':cms})
from wc2026_scraper import(scrape_summary,scrape_fixtures,scrape_standings,scrape_stats,scrape_wc_news,scrape_road_to_wc,get_wc2026_all,scrape_history,scrape_h2h,scrape_lineups,scrape_match_detail)
@app.get('/api/wc2026')
def _w():return JSONResponse(get_wc2026_all())
@app.get('/api/wc2026/fixtures')
def _wf():return JSONResponse(scrape_fixtures())
@app.get('/api/wc2026/standings')
def _ws():return JSONResponse(scrape_standings())
@app.get('/api/wc2026/stats')
def _wst():return JSONResponse(scrape_stats())
@app.get('/api/wc2026/history')
def _whi():return JSONResponse(scrape_history())
@app.get('/api/wc2026/news')
def _wn():return JSONResponse(scrape_wc_news())
@app.get('/api/wc2026/road')
def _wr():return JSONResponse(scrape_road_to_wc())
@app.get('/api/wc2026/h2h/{eid}')
def _wh2(eid:int):return JSONResponse(scrape_h2h(eid))
@app.get('/api/wc2026/lineups/{eid}')
def _wl(eid:int):return JSONResponse(scrape_lineups(eid))
@app.get('/api/wc2026/match/{eid}')
def _wm(eid:int):return JSONResponse(scrape_match_detail(eid))
def _bg():
    time.sleep(15)
    while True:
        try:get_wc2026_all()
        except:pass
        time.sleep(90)
threading.Thread(target=_bg,daemon=True).start()
app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static')