"""VNEWS v2 Entry Point - with fast bongda proxy"""
import sys, os
from main import app, HEADERS, BONGDA_HEADERS, fetch_bongda_api, HL_LEAGUES

try:
    import ai_ext
except Exception as e:
    print(f"[WARN] ai_ext import failed: {e}")

from fastapi.responses import HTMLResponse, JSONResponse, FileResponse, Response
from fastapi.staticfiles import StaticFiles
from starlette.routing import Mount
from fastapi import Query, Request, UploadFile, File, Form
import requests as req
from bs4 import BeautifulSoup
import re, html as html_lib, json, threading, time, uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import quote

HL_LEAGUES['friendly'] = {"path": "giai-khac/friendly", "name": "Giao hữu", "emoji": "🤝"}

STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/' and hasattr(r,'methods') and 'GET' in getattr(r,'methods',set()))]
app.routes[:]=[r for r in app.routes if not isinstance(r, Mount)]
app.router.routes=[r for r in app.router.routes if not isinstance(r, Mount)]

def _clean(s): return re.sub(r"\s+"," ",html_lib.unescape(str(s or""))).strip()

# Cache for match details (5 min TTL)
_match_cache = {}

# === FAST BONGDA PROXY ENDPOINT ===
def _get_match_detail(event_id, slug=None):
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept": "text/html", "Referer": "https://bongda.com.vn/"}
    if slug:
        url = f"https://bongda.com.vn/tran-dau/{event_id}/centre/{slug}"
    else:
        url = f"https://bongda.com.vn/tran-dau/{event_id}"
    resp = req.get(url, headers=headers, timeout=15, allow_redirects=True)
    if resp.status_code != 200:
        return None
    soup = BeautifulSoup(resp.text, 'html.parser')
    result = {"event_id": event_id, "found": False, "sections": []}
    info = {}
    tel = soup.select_one('.teams')
    if tel:
        he = tel.select_one('.team.home')
        if he:
            p_tags = [p for p in he.select('p') if not p.get('class') or 'logo' not in p.get('class', [])]
            if p_tags: info['home_team'] = _clean(p_tags[0].get_text())
            lo = he.select_one('img')
            if lo: info['home_logo'] = lo.get('src', '')
        ae = tel.select_one('.team.away')
        if ae:
            p_tags = ae.select('p')
            team_ps = [p for p in p_tags if not p.get('class') or 'logo' not in p.get('class', [])]
            if team_ps: info['away_team'] = _clean(team_ps[-1].get_text())
            lo = ae.select_one('img')
            if lo: info['away_logo'] = lo.get('src', '')
        sc = tel.select_one('.score')
        if sc:
            parts = [_clean(p.get_text()) for p in sc.select('p')]
            if len(parts) >= 2: info['score'] = f"{parts[0]} - {parts[1]}"
            lb = sc.select_one('.label')
            if lb: info['status_label'] = _clean(lb.get_text())
    if info.get('home_team') and info.get('away_team'):
        result['info'] = info
        result['found'] = True
        result['sections'].append('info')
    events = []
    for ev in soup.select('.events .period .event'):
        ev_cls = ' '.join(ev.get('class', []))
        ev_data = {'team': 'home' if 'home' in ev_cls else 'away', 'period': '', 'type': 'unknown', 'time': '', 'players': ''}
        parent = ev.parent
        if parent:
            h2 = parent.find('h2')
            if h2: ev_data['period'] = _clean(h2.get_text())
        if ev.select_one('[class*="goal"]'): ev_data['type'] = 'goal'
        elif ev.select_one('[class*="redcard"]'): ev_data['type'] = 'redcard'
        elif ev.select_one('[class*="yellowcard"]'): ev_data['type'] = 'yellowcard'
        elif ev.select_one('[class*="substitution"]'): ev_data['type'] = 'substitution'
        players_el = ev.select_one('.players')
        if players_el:
            pl_text = _clean(players_el.get_text(' ', strip=True))
            m = re.match(r"(\d+)'(.*)", pl_text)
            if m:
                ev_data['time'] = f"{m.group(1)}'"
                ev_data['players'] = m.group(2)
            else:
                ev_data['players'] = pl_text
        events.append(ev_data)
    if events:
        result['events'] = events
        result['sections'].append('events')
    pred = soup.select_one('.prediction-card')
    if pred:
        team_info = pred.select_one('.team-info')
        if team_info:
            teams = team_info.select('.team')
            pred_data = {}
            if len(teams) >= 2:
                pred_data['home_name'] = _clean(teams[0].select_one('.team-name').get_text()) if teams[0].select_one('.team-name') else ''
                pred_data['away_name'] = _clean(teams[1].select_one('.team-name').get_text()) if teams[1].select_one('.team-name') else ''
                divider = team_info.select_one('.divider')
                if divider: pred_data['result'] = _clean(divider.get_text())
                vc = pred.select_one('.vote-count')
                if vc: pred_data['vote_count'] = _clean(vc.get_text())
            result['prediction'] = pred_data
    recent = []
    ml = soup.select_one('.matches-list')
    if ml:
        for item in ml.select('.match-detail, .match-item, li'):
            de = item.select_one('.date, .time')
            le = item.select_one('.league')
            he_item = item.select_one('.home, .team-home')
            ae_item = item.select_one('.away, .team-away')
            se = item.select_one('.score, .result')
            if he_item or ae_item:
                recent.append({'date': _clean(de.get_text()) if de else '', 'league': _clean(le.get_text()) if le else '', 'home': _clean(he_item.get_text()) if he_item else '', 'away': _clean(ae_item.get_text()) if ae_item else '', 'score': _clean(se.get_text()) if se else 'vs'})
    if recent:
        result['recent_matches'] = recent
        result['sections'].append('recent')
    try:
        api_h = {"User-Agent": "Mozilla/5.0", "Accept": "application/json", "X-Requested-With": "XMLHttpRequest", "Referer": "https://bongda.com.vn/"}
        ar = req.get(f"https://bongda.com.vn/api/fixtures/h2h-stats?event_id={event_id}", headers=api_h, timeout=10)
        if ar.status_code == 200:
            ad = ar.json()
            if ad.get('status') == 'success' and ad.get('html'):
                asp = BeautifulSoup(ad['html'], 'html.parser')
                ast = {}
                for row in asp.select('li, tr'):
                    cells = row.select('td, span, p')
                    if len(cells) >= 3:
                        lb = _clean(cells[0].get_text())
                        if lb: ast[lb] = {'home': _clean(cells[1].get_text()), 'away': _clean(cells[2].get_text())}
                if ast:
                    result['h2h_stats_parsed'] = ast
                    result['sections'].append('h2h_stats')
    except: pass
    return result

@app.get('/api/proxy/bongda')
def proxy_bongda(event_id: int = Query(default=None), slug: str = Query(default=None)):
    if event_id is None:
        return JSONResponse({'error': 'event_id required'}, status_code=400)
    cache_key = f"{event_id}_{slug}"
    now = time.time()
    cached = _match_cache.get(cache_key)
    if cached and now - cached.get('_ts', 0) < 300:
        return JSONResponse(cached)
    try:
        result = _get_match_detail(event_id, slug)
        if result:
            result['_ts'] = now
            _match_cache[cache_key] = result
            return JSONResponse(result)
    except Exception as e:
        err = {"event_id": event_id, "found": False, "error": str(e), "_ts": now}
        _match_cache[cache_key] = err
        return JSONResponse(err)
    return JSONResponse({"event_id": event_id, "found": False})

@app.get('/api/match/{event_id}/detail')
def api_match_detail(event_id: int, url: str = Query(default=None)):
    slug = None
    if url:
        m = re.match(r'.+/tran-dau/\d+/(?:centre|preview)/(.+)', url)
        if m:
            slug = m.group(1)
    cache_key = f"{event_id}_{slug or ''}"
    now = time.time()
    cached = _match_cache.get(cache_key)
    if cached and now - cached.get('_ts', 0) < 300:
        return JSONResponse(cached)
    try:
        if not slug:
            try:
                home_r = req.get("https://bongda.com.vn/", headers={"User-Agent": "Mozilla/5.0"}, timeout=10)
                if home_r.status_code == 200:
                    home_soup = BeautifulSoup(home_r.text, 'html.parser')
                    for a in home_soup.select(f'a[href*="/tran-dau/{event_id}/"]'):
                        href = a.get('href', '')
                        m = re.match(r'/tran-dau/\d+/(?:centre|preview)/(.+)', href)
                        if m:
                            slug = m.group(1)
                            cache_key = f"{event_id}_{slug}"
                            break
            except: pass
        result = _get_match_detail(event_id, slug)
        if result:
            result['_ts'] = now
            _match_cache[cache_key] = result
            return JSONResponse(result)
    except Exception as e:
        err = {"event_id": event_id, "found": False, "error": str(e), "_ts": now}
        _match_cache[cache_key] = err
        return JSONResponse(err)
    return JSONResponse({"event_id": event_id, "found": False})

_STOP=set('và của các những một được trong với cho tại sau trước khi không người việt nam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì này đã để'.split())

def _has_kw(topic,title):
    tl=topic.lower();tt=(title or'').lower()
    if tl in tt:return True
    words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',tl) if len(w)>2 and w not in _STOP]
    if not words:return True
    return any(w in tt for w in words)

def _s_vnexpress(topic,limit=8):
    items=[]
    try:
        r=req.get(f"https://timkiem.vnexpress.net/?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for art in soup.select('article.item-news')[:limit]:
            a=art.select_one('h2 a, h3 a')
            if a and a.get('href'):
                t=_clean(a.get('title','') or a.get_text(strip=True))
                if _has_kw(topic,t):items.append({'title':t,'url':a['href'],'via':'VnExpress'})
    except:pass
    return items

def _s_dantri(topic,limit=8):
    items=[]
    try:
        r=req.get(f"https://dantri.com.vn/tim-kiem/{quote(topic)}.htm",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .article-title a[href]')[:limit*2]:
            t=_clean(a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://dantri.com.vn'+href
                items.append({'title':t,'url':href,'via':'Dân Trí'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_vietnamnet(topic,limit=6):
    items=[]
    try:
        r=req.get(f"https://vietnamnet.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .vnn-title a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://vietnamnet.vn'+href
                items.append({'title':t,'url':href,'via':'VietNamNet'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_bongda(topic,limit=5):
    items=[]
    try:
        r=req.get(f"https://bongda.com.vn/tim-kiem.html?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .title a[href]')[:limit*2]:
            t=_clean(a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://bongda.com.vn'+href
                items.append({'title':t,'url':href,'via':'Bóng Đá'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_genk(topic,limit=5):
    items=[]
    try:
        r=req.get(f"https://genk.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('a[href$=".chn"]')[:limit*3]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if href.startswith('/'):href='https://genk.vn'+href
                items.append({'title':t,'url':href,'via':'GenK'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_thanhnien(topic,limit=6):
    items=[]
    try:
        r=req.get(f"https://thanhnien.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .box-title a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://thanhnien.vn'+href
                items.append({'title':t,'url':href,'via':'Thanh Niên'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_tuoitre(topic,limit=6):
    items=[]
    try:
        r=req.get(f"https://tuoitre.vn/tim-kiem.htm?keywords={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .box-title-text a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://tuoitre.vn'+href
                items.append({'title':t,'url':href,'via':'Tuổi Trẻ'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_thethaovanhoa(topic,limit=5):
    items=[]
    try:
        r=req.get(f"https://thethaovanhoa.vn/tim-kiem.htm?keyword={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .title a[href]')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://thethaovanhoa.vn'+href
                items.append({'title':t,'url':href,'via':'TT&VH'})
            if len(items)>=limit:break
    except:pass
    return items

def _search_all(topic,limit=36):
    results={}
    with ThreadPoolExecutor(8) as ex:
        futs={ex.submit(_s_vnexpress,topic,8):'vne',ex.submit(_s_dantri,topic,8):'dt',ex.submit(_s_vietnamnet,topic,6):'vnn',ex.submit(_s_bongda,topic,5):'bd',ex.submit(_s_genk,topic,5):'gk',ex.submit(_s_thanhnien,topic,6):'tn',ex.submit(_s_tuoitre,topic,6):'tt',ex.submit(_s_thethaovanhoa,topic,5):'tvh'}
        for f in as_completed(futs,timeout=14):
            try:results[futs[f]]=f.result()
            except:results[futs[f]]=[]
    srcs=list(results.values());out=[];seen=set()
    for i in range(max((len(s) for s in srcs),default=0)):
        for s in srcs:
            if i<len(s) and s[i].get('url') and s[i]['url'] not in seen:seen.add(s[i]['url']);out.append(s[i])
    return out[:limit]

# Remove main.py routes that app_v2_entry overrides (main.py registers first, FastAPI uses first match)
for _path in ['/api/article', '/api/hot_topics', '/api/categories', '/api/storage_status', '/s']:
    app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)==_path and 'GET' in getattr(r,'methods',set()))]

# ===== Article cache (TTL 30 min, keyed by URL) =====
_article_cache = {}
_article_cache_ttl = 1800

# Dedicated session for article scraping (no rate limiter — we only scrape one article at a time per request)
_art_session = None
_art_lock = threading.Lock()
def _get_art_session():
    global _art_session
    if _art_session is None:
        with _art_lock:
            if _art_session is None:
                _art_session = req.Session()
                _art_session.headers.update({
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
                    "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8",
                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                })
    return _art_session

def _scrape_article_fast(url):
    """Fast article scrape — single request, no rate limiter, fail fast with OG fallback."""
    from urllib.parse import urlparse
    domain = urlparse(url).netloc
    sess = _get_art_session()
    
    # Try mobile UA first (lighter HTML), then desktop
    uas = [
        {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"},
        {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"},
    ]
    
    for ua in uas:
        try:
            r = sess.get(url, headers=ua, timeout=6, allow_redirects=True)
            if not r or r.status_code != 200:
                continue
            r.encoding = 'utf-8'
            soup = BeautifulSoup(r.text, 'lxml')
            
            # Remove junk
            for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe','.ads','.ad','.banner-ads','.fb-comments','.fb-root','.social-share','.related-news','.tag','.breadcrumb']):
                tag.decompose()
            
            # Extract OG meta
            title = summary = og_img = ""
            ogt = soup.find('meta', property='og:title')
            if ogt: title = ogt.get('content', '')
            ogd = soup.find('meta', property='og:description') or soup.find('meta', attrs={'name': 'description'})
            if ogd: summary = ogd.get('content', '')[:500]
            ogi = soup.find('meta', property='og:image')
            if ogi:
                og_img = ogi.get('content', '')
                if og_img.startswith('//'): og_img = 'https:' + og_img
            h1 = soup.find('h1')
            if not title and h1: title = h1.get_text(strip=True)[:200]
            
            # Try to find article body
            body = []
            selectors = [
                '.fck_detail', '.sidebar-1',  # VnExpress
                '.singular-content', '.dt__content', '.article-content', '.content-detail', '#divNewsContent',  # Dân Trí
                '.content-detail', '.main-content-detail', '.box-content',  # Tuổi Trẻ
                '.knc-content', '.article-body', '.detail-body',  # Kenh14/GenK
                '.article-detail', '.detail-content',  # Thanh Niên
                'article', 'main', '.cms-body', '.article__body', '.post-content',
                '.entry-content', '#content', '.article-text', '.story-body',
            ]
            for sel in selectors:
                el = soup.select_one(sel)
                if el and len(el.find_all('p')) >= 2:
                    seen_imgs = set()
                    for child in el.find_all(['p','h2','h3','figure','img'], recursive=True):
                        if child.name == 'p':
                            t = child.get_text(strip=True)
                            if t and len(t) > 15:
                                body.append({'type': 'p', 'text': t})
                        elif child.name in ('h2','h3'):
                            t = child.get_text(strip=True)
                            if t:
                                body.append({'type': 'heading', 'text': t})
                        elif child.name in ('figure','img'):
                            im = child if child.name == 'img' else child.find('img')
                            if im:
                                src = im.get('data-src') or im.get('src') or im.get('data-lazy') or ''
                                if src and 'base64' not in src and src not in seen_imgs:
                                    seen_imgs.add(src)
                                    if src.startswith('//'): src = 'https:' + src
                                    body.append({'type': 'img', 'src': src})
                        if child.name == 'figure':
                            cap = child.find('figcaption')
                            if cap:
                                ct = cap.get_text(strip=True)
                                if ct: body.append({'type': 'p', 'text': ct})
                    if len(body) >= 2:
                        return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img,
                                'body': body[:50], 'source': domain, 'url': url}
            
            # No body found — use OG meta as fallback
            if title and (summary or og_img):
                fallback = []
                if og_img: fallback.append({'type': 'img', 'src': og_img})
                if summary: fallback.append({'type': 'p', 'text': summary})
                if fallback:
                    return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img,
                            'body': fallback, 'source': domain, 'url': url, 'fallback': True}
            
            # Got HTML but no body and no OG — return title at least
            if title:
                return {'title': _clean(title), 'summary': '', 'og_image': '',
                        'body': [{'type': 'p', 'text': 'Nội dung đang được tải...'}],
                        'source': domain, 'url': url, 'fallback': True}
            
            break  # Got 200 but no content at all — don't retry other UA
        except Exception:
            continue
    
    return None

@app.get('/api/article')
def api_article_v2(url: str = Query(...)):
    """Scrape article and return JSON for VNEWS SPA. Fast, cached, with fallback."""
    from urllib.parse import unquote
    safe_url = unquote(url)
    
    try:
        # Check cache first
        now = time.time()
        cached = _article_cache.get(safe_url)
        if cached and now - cached['t'] < _article_cache_ttl:
            resp = JSONResponse(cached['d'])
            resp.headers["Cache-Control"] = "public, max-age=1800"
            return resp
        
        # Fetch fresh — use fast scraper for ALL sites (simpler, more reliable)
        data = _scrape_article_fast(safe_url)
        
        if data and data.get('body'):
            _article_cache[safe_url] = {'d': data, 't': now}
            resp = JSONResponse(data)
            resp.headers["Cache-Control"] = "public, max-age=1800"
            return resp
        
        # Last resort: try RSS fallback
        try:
            from main import _fetch_rss_fallback
            from urllib.parse import urlparse as _up
            rss_data = _fetch_rss_fallback(safe_url, _up(safe_url).netloc)
            if rss_data and rss_data.get('title'):
                body = []
                if rss_data.get('og_image'):
                    body.append({'type': 'img', 'src': rss_data['og_image']})
                if rss_data.get('summary'):
                    sentences = re.split(r'(?<=[.!?])\s+', rss_data['summary'])
                    for s in sentences[:10]:
                        if len(s.strip()) > 20:
                            body.append({'type': 'p', 'text': s.strip()})
                if body:
                    result = {
                        'title': rss_data['title'], 'summary': rss_data['summary'][:500],
                        'og_image': rss_data.get('og_image', ''), 'body': body[:50],
                        'source': 'rss', 'url': safe_url, 'fallback': True, 'rss': True
                    }
                    _article_cache[safe_url] = {'d': result, 't': now}
                    resp = JSONResponse(result)
                    resp.headers["Cache-Control"] = "public, max-age=600"
                    return resp
        except Exception:
            pass
        
        result = {'error': 'Không đọc được', 'url': safe_url}
        resp = JSONResponse(result)
        resp.headers["Cache-Control"] = "public, max-age=60"
        return resp
    except Exception as e:
        import traceback
        tb = traceback.format_exc()
        return JSONResponse({'error': f'Server error: {str(e)[:100]}', 'trace': tb[-500:], 'url': safe_url}, status_code=200)

_hot_cache={'t':0,'d':[]}
def _get_hot_topics():
    now=time.time()
    if _hot_cache['d'] and now-_hot_cache['t']<600:return _hot_cache['d']
    freq={};display={}
    feeds=['https://vnexpress.net/rss/tin-moi-nhat.rss','https://dantri.com.vn/rss/home.rss','https://vietnamnet.vn/rss/tin-moi-nhat.rss','https://thanhnien.vn/rss/home.rss','https://tuoitre.vn/rss/tin-moi-nhat.rss','https://genk.vn/rss','https://vnexpress.net/rss/the-thao.rss','https://thethaovanhoa.vn/rss/tin-nong.rss']
    for feed_url in feeds:
        try:
            r=req.get(feed_url,headers={'User-Agent':'Mozilla/5.0'},timeout=6);r.encoding='utf-8';soup=BeautifulSoup(r.text,'xml')
            for item in soup.find_all('item')[:12]:
                title=_clean(item.find('title').get_text() if item.find('title') else '')
                if not title:continue
                title=re.sub(r'\s*[-|].*$','',title);words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP]
                if len(words)<2:continue
                for n in(3,4,2):
                    for i in range(max(0,len(words)-n+1)):
                        phrase=' '.join(words[i:i+n])
                        if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase
        except:continue
    ranked=sorted(freq.items(),key=lambda x:x[1],reverse=True);topics=[];seen=set()
    for key,count in ranked:
        is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen)
        if is_dup:continue
        seen.add(key);topics.append({'label':'#'+re.sub(r'\s+','',display[key].title()),'topic':display[key],'count':count})
        if len(topics)>=20:break
    for kw in['World Cup 2026','Kinh tế Việt Nam','Bóng đá châu Âu','Công nghệ AI','Giá vàng','Thời tiết']:
        if len(topics)>=24:break
        if not any(kw.lower() in s for s in seen):topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw,'count':0})
    _hot_cache.update({'t':now,'d':topics[:24]});return topics[:24]

@app.get('/api/hot_topics')
def api_hot_topics():
    resp = JSONResponse({'topics':_get_hot_topics()})
    resp.headers["Cache-Control"] = "public, max-age=120"
    return resp
@app.get('/')
async def serve_index():
    p=os.path.join(STATIC_DIR,'index_v2.html')
    if os.path.exists(p):return FileResponse(p,media_type='text/html')
    return HTMLResponse('<h1>VNEWS</h1>')
@app.get('/api/hashtag/sources')
def _ht(topic:str=Query(...),page:int=Query(default=0)):
    items=_search_all(topic,36);per_page=8;start=page*per_page;end=start+per_page
    return JSONResponse({'sources':items[start:end],'topic':topic,'page':page,'has_more':end<len(items),'total':len(items)})
@app.get('/api/categories')
def _cat():return JSONResponse([])
@app.get('/api/storage_status')
def _st():return JSONResponse({'persistent':os.path.isdir('/data') and os.access('/data',os.W_OK)})
@app.get('/s')
async def _sh(url:str='',title:str='',img:str=''):return HTMLResponse(f'<!DOCTYPE html><html><head><meta property="og:title" content="{_clean(title)}"><meta property="og:image" content="{_clean(img)}"><meta http-equiv="refresh" content="0;url={_clean(url) or "/"}"></head><body></body></html>')

from wc2026_scraper import(scrape_summary,scrape_fixtures,scrape_standings,scrape_stats,scrape_wc_news,scrape_road_to_wc,get_wc2026_all,scrape_history,scrape_h2h,scrape_lineups,scrape_match_detail)

# === XEMLAIBONGDA PROXY (CORS workaround for WC highlights) ===
_xlb_cache = {}
_xlb_lock = threading.Lock()

def _xlb_scrape(path):
    url = f"https://xemlaibongda.top/{path}"
    r = req.get(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}, timeout=15, allow_redirects=True)
    if r.status_code != 200:
        return []
    soup = BeautifulSoup(r.text, 'lxml')
    vids = []
    seen = set()
    for a in soup.select('a[href*="/video/"]'):
        href = a.get('href', '')
        if not href or href in seen:
            continue
        seen.add(href)
        if not href.startswith('http'):
            href = 'https://xemlaibongda.top' + href
        img = a.select_one('img')
        p = a.parent
        for _ in range(4):
            if img:
                break
            if p:
                img = p.select_one('img')
                p = p.parent
        img_src = ''
        if img:
            img_src = img.get('data-src','') or img.get('src','') or img.get('data-lazy','') or img.get('data-original','')
            if img_src.startswith('//'):
                img_src = 'https:' + img_src
            elif img_src.startswith('/'):
                img_src = 'https://xemlaibongda.top' + img_src
        title = ''
        for sel in ['.title', 'h3', 'h2', '.name', '.post-title', '.entry-title', '.video-title']:
            t = a.select_one(sel)
            if t:
                title = _clean(t.get_text())
                break
        if not title:
            title = _clean(a.get('title',''))
        if not title:
            img_alt = a.select_one('img')
            if img_alt:
                title = _clean(img_alt.get('alt',''))
        if not title:
            parent = a.parent
            if parent:
                pt = _clean(parent.get_text(' ',strip=True))
                if 5 < len(pt) < 120:
                    title = pt
        if not title or len(title) < 3:
            continue
        vids.append({"link": href, "img": img_src, "title": title})
        if len(vids) >= 30:
            break
    return vids

@app.get('/api/proxy/xlb')
def proxy_xlb(path: str = Query(default="")):
    now = time.time()
    cache_key = f"xlb:{path}"
    with _xlb_lock:
        cached = _xlb_cache.get(cache_key)
        if cached and now - cached['t'] < 120:
            return JSONResponse(cached['d'])
    try:
        vids = _xlb_scrape(path)
        result = {"videos": vids, "count": len(vids)}
        with _xlb_lock:
            _xlb_cache[cache_key] = {'t': now, 'd': result}
        return JSONResponse(result)
    except Exception as e:
        return JSONResponse({"videos": [], "count": 0, "error": str(e)}, status_code=500)

@app.get('/api/wc2026')
def _w():return JSONResponse(get_wc2026_all())
@app.get('/api/wc2026/fixtures')
def _wf():return JSONResponse(scrape_fixtures())
@app.get('/api/wc2026/standings')
def _ws():return JSONResponse(scrape_standings())
@app.get('/api/wc2026/stats')
def _wst():return JSONResponse(scrape_stats())
@app.get('/api/wc2026/history')
def _whi():return JSONResponse(scrape_history())
@app.get('/api/wc2026/news')
def _wn():return JSONResponse(scrape_wc_news())
@app.get('/api/wc2026/road')
def _wr():return JSONResponse(scrape_road_to_wc())
@app.get('/api/wc2026/h2h/{eid}')
def _wh2(eid:int):return JSONResponse(scrape_h2h(eid))
@app.get('/api/wc2026/lineups/{eid}')
def _wl(eid:int):return JSONResponse(scrape_lineups(eid))
@app.get('/api/wc2026/match/{eid}')
def _wm(eid:int):return JSONResponse(scrape_match_detail(eid))

DATA_DIR='/data' if os.path.isdir('/data') else os.path.join(os.path.dirname(os.path.abspath(__file__)),'data')
os.makedirs(DATA_DIR,exist_ok=True)
IF=os.path.join(DATA_DIR,'interactions_v2.json')
CF=os.path.join(DATA_DIR,'comments_v2.json')
WALL_FILE=os.path.join(DATA_DIR,'wall_posts.json')
WALL_VIDEO_DIR=os.path.join(DATA_DIR,'wall_videos')
os.makedirs(WALL_VIDEO_DIR,exist_ok=True)

_il=threading.Lock();_cl=threading.Lock();_wl_lock=threading.Lock()
def _lj(p):
    try:
        if os.path.exists(p):return json.load(open(p,'r',encoding='utf-8'))
    except:pass
    return{}
def _sj(p,d):
    try:open(p+'.tmp','w',encoding='utf-8').write(json.dumps(d,ensure_ascii=False));os.replace(p+'.tmp',p)
    except:pass

@app.post('/api/v2/interact')
async def _int(request:Request):
    b=await request.json();v=str(b.get('id','')).strip();t=str(b.get('type','')).strip()
    if not v or t not in('view','like'):return JSONResponse({'error':'x'},status_code=400)
    with _il:db=_lj(IF);db.setdefault(v,{'views':0,'likes':0,'comments':0});db[v][t+'s']+=1;_sj(IF,db);return JSONResponse(db[v])

@app.get('/api/v2/interactions')
def _gi(id:str=Query(...)):
    with _il:return JSONResponse(_lj(IF).get(id.strip(),{'views':0,'likes':0,'comments':0}))

@app.get('/api/v2/comments')
def _gc(id:str=Query(...)):
    with _cl:return JSONResponse({'comments':_lj(CF).get(id.strip(),[])})

@app.post('/api/v2/comment')
async def _pc(request:Request):
    b=await request.json();v=str(b.get('id','')).strip();tx=str(b.get('text','')).strip()[:500]
    if not v or not tx:return JSONResponse({'error':'x'},status_code=400)
    c={'text':tx,'time':time.strftime('%H:%M %d/%m',time.localtime()),'ts':int(time.time())}
    with _cl:db=_lj(CF);db.setdefault(v,[]);db[v].append(c);db[v]=db[v][-200:];_sj(CF,db);cms=db[v]
    with _il:idb=_lj(IF);idb.setdefault(v,{'views':0,'likes':0,'comments':0});idb[v]['comments']=len(cms);_sj(IF,idb)
    return JSONResponse({'comments':cms})

# ===== WALL / SHORT AI ENDPOINTS =====

def _load_wall_posts():
    """Load wall posts from JSON file."""
    with _wl_lock:
        return _lj(WALL_FILE)

def _save_wall_posts(posts):
    """Save wall posts to JSON file."""
    with _wl_lock:
        _sj(WALL_FILE, posts)

@app.get('/api/wall')
def api_wall():
    """Get all wall posts."""
    posts = _load_wall_posts()
    if not posts:
        # Return empty list, not error
        return JSONResponse({"posts": []})
    return JSONResponse({"posts": posts})

@app.post('/api/wall')
async def api_wall_post(request: Request):
    """
    Create a wall post. Supports:
    - JSON body: {title, text, img, source}
    - Multipart form: title, text, source + video file upload
    """
    content_type = request.headers.get('content-type', '')

    # Handle multipart upload (video file)
    if 'multipart/form-data' in content_type:
        try:
            form = await request.form()
        except Exception as e:
            return JSONResponse({"error": f"Form parse error: {str(e)}"}, status_code=400)

        title = form.get('title', 'Video mới') or 'Video mới'
        text = form.get('text', '') or ''
        source = form.get('source', 'vtv_recorder') or 'vtv_recorder'
        video_file = form.get('video')

        post_id = str(uuid.uuid4())[:12]
        video_url = None

        # Save video file if provided
        if video_file and hasattr(video_file, 'filename') and video_file.filename:
            # Determine extension
            fname = video_file.filename.lower()
            if fname.endswith('.mp4'):
                ext = '.mp4'
            elif fname.endswith('.webm'):
                ext = '.webm'
            else:
                ext = '.webm'

            video_filename = f"wall_{post_id}{ext}"
            video_path = os.path.join(WALL_VIDEO_DIR, video_filename)

            try:
                # Read file content
                content = await video_file.read()
                if not content:
                    return JSONResponse({"error": "Empty video file"}, status_code=400)

                # Save to disk
                with open(video_path, 'wb') as f:
                    f.write(content)

                file_size_mb = len(content) / 1024 / 1024
                if file_size_mb > 50:
                    os.remove(video_path)
                    return JSONResponse({"error": f"Video quá lớn ({file_size_mb:.1f}MB). Tối đa 50MB."}, status_code=400)

                # URL to access the video
                video_url = f"/api/wall/video/{video_filename}"
            except Exception as e:
                return JSONResponse({"error": f"Lỗi lưu video: {str(e)}"}, status_code=500)

        # Create post
        post = {
            "id": post_id,
            "title": title[:200],
            "text": text[:2000],
            "source": source,
            "video": video_url,
            "img": None,
            "images": [],
            "created": int(time.time()),
            "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()),
        }

        # Save to wall
        posts = _load_wall_posts()
        if not isinstance(posts, list):
            posts = []
        posts.insert(0, post)
        # Keep max 200 posts
        posts = posts[:200]
        _save_wall_posts(posts)

        return JSONResponse({"post": post, "ok": True})

    # Handle JSON body (text-only post)
    try:
        body = await request.json()
    except:
        body = {}

    title = body.get('title', 'Bài mới') or 'Bài mới'
    text = body.get('text', '') or ''
    img = body.get('img', None)
    source = body.get('source', 'user') or 'user'

    post_id = str(uuid.uuid4())[:12]
    post = {
        "id": post_id,
        "title": title[:200],
        "text": text[:2000],
        "source": source,
        "video": None,
        "img": img,
        "images": [],
        "created": int(time.time()),
        "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()),
    }

    posts = _load_wall_posts()
    if not isinstance(posts, list):
        posts = []
    posts.insert(0, post)
    posts = posts[:200]
    _save_wall_posts(posts)

    return JSONResponse({"post": post, "ok": True})

@app.get('/api/wall/video/{filename}')
def api_wall_video(filename: str):
    """Serve a wall video file."""
    # Security: prevent path traversal
    if '..' in filename or '/' in filename:
        return Response(status_code=403)
    video_path = os.path.join(WALL_VIDEO_DIR, filename)
    if not os.path.exists(video_path):
        return Response(status_code=404)
    ext = os.path.splitext(filename)[1].lower()
    media_type = 'video/mp4' if ext == '.mp4' else 'video/webm'
    return FileResponse(video_path, media_type=media_type)

@app.delete('/api/wall/{post_id}')
def api_wall_delete(post_id: str):
    """Delete a wall post and its video."""
    posts = _load_wall_posts()
    if not isinstance(posts, list):
        return JSONResponse({"error": "No posts"}, status_code=404)

    for i, p in enumerate(posts):
        if p.get('id') == post_id:
            # Delete video file if exists
            if p.get('video'):
                video_name = p['video'].split('/')[-1]
                video_path = os.path.join(WALL_VIDEO_DIR, video_name)
                if os.path.exists(video_path):
                    os.remove(video_path)
            posts.pop(i)
            _save_wall_posts(posts)
            return JSONResponse({"ok": True})

    return JSONResponse({"error": "Post not found"}, status_code=404)

def _bg():
    time.sleep(15)
    while True:
        try:get_wc2026_all()
        except:pass
        time.sleep(90)
threading.Thread(target=_bg,daemon=True).start()

app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static')