main.py

"""VNEWS - FastAPI backend with livescore + xemlaibongda highlights + YouTube VTV shorts"""
import re, time, subprocess, json, os, threading
import html as html_lib
from datetime import datetime, timezone, timedelta
from collections import defaultdict

VN_TZ = timezone(timedelta(hours=7))
from concurrent.futures import ThreadPoolExecutor, as_completed
from fastapi import FastAPI, Query, Request
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse, Response
from urllib.parse import quote
import requests
from bs4 import BeautifulSoup

app = FastAPI()

# ===== RATE LIMITING =====app = FastAPI()

# ===== WORLD CUP 2026 SCRAPER =====
from wc2026_scraper import get_wc2026_all, scrape_fixtures, scrape_standings, scrape_stats, scrape_wc_news

# ===== RATE LIMITING =====
_rate_limit_data = defaultdict(list)  # {ip: [timestamp1, timestamp2, ...]}
_rate_limit_lock = threading.Lock()
RATE_LIMIT_MAX = 60  # Max requests per minute per IP
RATE_LIMIT_WINDOW = 60  # seconds

def _check_rate_limit(ip: str) -> bool:
    """Kiểm tra rate limit, return True nếu OK, False nếu bị limit"""
    with _rate_limit_lock:
        now = time.time()
        # Xóa các request cũ
        _rate_limit_data[ip] = [t for t in _rate_limit_data[ip] if now - t < RATE_LIMIT_WINDOW]
        if len(_rate_limit_data[ip]) >= RATE_LIMIT_MAX:
            return False
        _rate_limit_data[ip].append(now)
        return True

@app.middleware("http")
async def rate_limit_middleware(request: Request, call_next):
    """Middleware để kiểm tra rate limit"""
    # Chỉ rate limit API endpoints
    if request.url.path.startswith("/api/"):
        ip = request.client.host
        if not _check_rate_limit(ip):
            return JSONResponse({"error": "rate limit exceeded"}, status_code=429)
    response = await call_next(request)
    return response

# ===== VTV CHANNELS API =====
from vtv_api import router as vtv_router
app.include_router(vtv_router)

HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","Accept-Language":"vi-VN,vi;q=0.9,en;q=0.8"}
BONGDA_HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"vi-VN,vi;q=0.9","Referer":"https://bongda.com.vn/lich-thi-dau","X-Requested-With":"XMLHttpRequest"}
BASE_BDP = "https://bongdaplus.vn"
_cache = {}
_cache_ttl = 300
_cache_ttl_live = 60
_cache_ttl_yt = 1800

# ===== VTV NAM BO SHORTS FALLBACK =====
SHORTS_FALLBACK = [
    {"id":"nqlLH6chLRo","title":"Tin nóng VTV Nam Bộ | #shorts","channel":"vtvnambo"},
    {"id":"E7Kq0v3hG6w","title":"VTV Nam Bộ - Tin tức miền Nam | #shorts","channel":"vtvnambo"},
    {"id":"Lu_iCQ5YwNM","title":"Công an lập hồ sơ xử lý người phụ nữ chửi bới tát nam tài xế ô tô ở Hà Nội","channel":"baodantri7941"},
    {"id":"CwWvijF8BOA","title":"Chú rể Ninh Bình bật khóc nhận món quà bí mật người cha quá cố gửi 26 năm trước","channel":"baodantri7941"},
    {"id":"tvPewsc2ph4","title":"Tính năng ẩn trên iPhone giúp giảm mỏi mắt","channel":"baodantri7941"},
    {"id":"b1Nxzv9ixlU","title":"Y án 3 năm tù với nữ tài xế uống 8 lon bia lái xe tông chủ tịch xã tử vong","channel":"baodantri7941"},
    {"id":"Xp5eTwAZAis","title":"Người đánh hàng xóm tại chung cư ở Hà Nội bị tuyên hơn 4 tháng tù","channel":"baodantri7941"},
    {"id":"Htzvwg6iOBM","title":"Xe điện Audi S6 Sportback e-tron có gì đặc biệt?","channel":"baodantri7941"},
    {"id":"iMdFmWvYdlo","title":"Cô gái người Nga yêu thời trang và đất nước Việt Nam","channel":"baodantri7941"},
    {"id":"IVaRc6moEv8","title":"Người nông dân Trung Quốc đột quỵ bệnh viện giúp bán sạch 4 tấn táo","channel":"baodantri7941"},
    {"id":"uVxqPxToItU","title":"Công an vào cuộc vụ người phụ nữ chửi bới hành hung tài xế ô tô ở Hà Nội","channel":"baodantri7941"},
    {"id":"VAfgNNgZDRs","title":"Khởi tố 4 đối tượng ném bom xăng vào nhà dân ở Đồng Nai","channel":"baodantri7941"},
    {"id":"sBH_-zGh0Xw","title":"Vì sao Times New Roman vẫn nổi tiếng sau hàng chục năm?","channel":"baodantri7941"},
    {"id":"woKn5f2bLHM","title":"Quảng Ninh ngập sâu diện rộng sau đợt mưa lớn","channel":"baodantri7941"},
    {"id":"bcpgRoxbLPw","title":"Giông lốc quật bay mái tôn ở TP.HCM","channel":"baodantri7941"},
    {"id":"ZIIC5osy544","title":"Bé trai Trung Quốc rơi từ tầng 11 vẫn sống sót kỳ diệu","channel":"baodantri7941"},
    {"id":"uTMJ49NQpyc","title":"Sau lớp mascot 40kg Câu chuyện mưu sinh của người trẻ ở TPHCM","channel":"baodantri7941"},
    {"id":"7Pd6vZ2Lz1M","title":"Hành động ấm lòng của người đàn ông tìm kiếm 5 học sinh tử vong ở sông Lô","channel":"baosuckhoedoisongboyte"},
    {"id":"SlHLt_ZyPiE","title":"Xử phạt người đàn ông xóa số điện thoại cứu hộ trên cao tốc Bắc Nam","channel":"baosuckhoedoisongboyte"},
    {"id":"IUOprcJyYr4","title":"Phụ nữ táo bón có phải do lười ăn rau?","channel":"baosuckhoedoisongboyte"},
    {"id":"YY8ojFNE-AU","title":"Quái xế tự quay clip nẹt pô đánh võng đăng TikTok bị xử lý","channel":"baosuckhoedoisongboyte"},
    {"id":"OV7_oGdQGII","title":"Bố cô dâu khóc sụt sùi rồi quẩy cực sung gây bão mạng","channel":"baosuckhoedoisongboyte"},
    {"id":"FoxhFyz2skY","title":"Người đàn ông nước ngoài đập phá ô tô bẻ cần gạt nước ở Đà Nẵng","channel":"baosuckhoedoisongboyte"},
    {"id":"R1oC_I8dFPU","title":"Thanh niên buông tay lái đứng trên xe máy khi đổ đèo ở Đắk Lắk","channel":"baosuckhoedoisongboyte"},
    {"id":"U0Ft6ChWAIo","title":"Cô giáo kể phút tháo chạy khỏi xe khách trước khi bị lũ vò nát ở Cao Bằng","channel":"baosuckhoedoisongboyte"},
    {"id":"hH0ANeze_4E","title":"Liên tiếp hàng chục con bò bị sét đánh chết trong ngày mưa dông","channel":"baosuckhoedoisongboyte"},
    {"id":"pXWt0QbAzRQ","title":"Va chạm giao thông người phụ nữ lăng mạ tài xế ô tô","channel":"baosuckhoedoisongboyte"},
    {"id":"UWWLPY1OYt4","title":"CSGT chặn xe khách khống chế đối tượng cướp dây chuyền tại Gia Lai","channel":"baosuckhoedoisongboyte"},
    {"id":"AxhVTQutsuo","title":"Xuất tinh sớm và những hiểu lầm thường gặp","channel":"baosuckhoedoisongboyte"},
    {"id":"cNy6FgaNxYM","title":"Cô dâu khóc sưng mắt vì 6 chỉ vàng không cánh mày bay trong ngày cưới","channel":"baosuckhoedoisongboyte"},
    {"id":"IDt_S6q59Ro","title":"Chở bạn gái không đội mũ bảo hiểm thanh niên đấm CSGT","channel":"baosuckhoedoisongboyte"},
    {"id":"LFxJ9Ik6W0A","title":"Mệnh lệnh từ trái tim CSGT Hà Nội mở đường đưa bé 5 tháng tuổi đi cấp cứu","channel":"baosuckhoedoisongboyte"},
]
for _v in SHORTS_FALLBACK:
    _v.setdefault("link", "https://www.youtube.com/watch?v="+_v["id"])
    _v.setdefault("img", "https://i.ytimg.com/vi/"+_v["id"]+"/hqdefault.jpg")
    _v.setdefault("source", "yt")

SHORT_STATS_FILE = "/data/short_stats.json" if os.path.isdir("/data") else "/app/short_stats.json"
_short_lock = threading.Lock()
def _load_short_db():
    try:
        if os.path.exists(SHORT_STATS_FILE):
            with open(SHORT_STATS_FILE,"r",encoding="utf-8") as f: return json.load(f)
    except: pass
    return {}
def _save_short_db(db):
    try:
        os.makedirs(os.path.dirname(SHORT_STATS_FILE), exist_ok=True)
        tmp = SHORT_STATS_FILE + ".tmp"
        with open(tmp,"w",encoding="utf-8") as f: json.dump(db, f, ensure_ascii=False)
        os.replace(tmp, SHORT_STATS_FILE)
    except: pass
def _short_default(): return {"views":0,"likes":0,"shares":0,"comments":[]}

PRIORITY_LEAGUES = ["Ngoại Hạng Anh","FA Cup","Champions League","LaLiga","Copa del Rey","Serie A","Bundesliga","Ligue 1","V-League"]
LEAGUE_IDS = {"nha":27110,"laliga":27233,"seriea":27044,"bundesliga":26891,"ligue1":27212}
HL_LEAGUES = {
    "premier-league":{"path":"anh/premier-league","name":"Premier League","emoji":"🏴󠁧󠁢󠁥󠁮󠁧󠁿"},
    "fa-cup":{"path":"anh/fa-cup","name":"FA Cup","emoji":"🏆"},
    "bundesliga":{"path":"duc/bundesliga","name":"Bundesliga","emoji":"🇩🇪"},
    "serie-a":{"path":"italy/serie-a","name":"Serie A","emoji":"🇮🇹"},
    "la-liga":{"path":"tay-ban-nha/la-liga","name":"La Liga","emoji":"🇪🇸"},
    "champions-league":{"path":"cup-chau-au/uefa-champions-league","name":"Champions League","emoji":"⭐"},
    "europa-league":{"path":"cup-chau-au/uefa-europa-league","name":"Europa League","emoji":"🟠"},
    "world-cup":{"path":"the-gioi/world-cup","name":"World Cup 2026","emoji":"🌍"},
}
def _cached(key, fn, ttl=None):
    now=time.time(); t=ttl or _cache_ttl
    if key in _cache and now-_cache[key]["t"]<t: return _cache[key]["d"]
    try: data=fn()
    except: data=_cache.get(key,{}).get("d",[])
    _cache[key]={"d":data,"t":now}; return data
def _get(url, headers=None):
    h=headers or HEADERS; r=requests.get(url, headers=h, timeout=15); r.encoding="utf-8"
    return BeautifulSoup(r.text,"lxml")
def fetch_bongda_api(endpoint):
    try:
        r=requests.get(f"https://bongda.com.vn{endpoint}", headers=BONGDA_HEADERS, timeout=10)
        if r.status_code==200:
            data=r.json()
            if data.get("status")=="success": return data.get("html","")
        return ""
    except: return ""

def _parse_match_from_li(li, status_type="live"):
    match_div=li.select_one("div.match")
    if not match_div: return None
    home_el=match_div.select_one(".home-team .name"); away_el=match_div.select_one(".away-team .name")
    if not home_el or not away_el: return None
    status_el=match_div.select_one(".status a"); league_el=li.find_previous("strong"); time_el=match_div.select_one(".match-time")
    home_logo=match_div.select_one(".home-team .logo img"); away_logo=match_div.select_one(".away-team .logo img")
    event_id=""
    if status_el:
        href=status_el.get("href",""); m=re.search(r'/tran-dau/(\d+)/',href)
        if m: event_id=m.group(1)
    spans=status_el.find_all("span") if status_el else []; score=""; minute=""
    if len(spans)>=3: score=f"{spans[0].get_text(strip=True)} - {spans[2].get_text(strip=True)}"
    if len(spans)>=4: minute=spans[3].get_text(strip=True)
    if not score and status_el and status_el.select_one(".vs"): score="VS"
    league=league_el.get_text(strip=True) if league_el else ""
    return {"home":home_el.get_text(strip=True),"away":away_el.get_text(strip=True),"score":score or"VS","minute":minute,"league":league,"time":time_el.get_text(strip=True) if time_el else "","event_id":event_id,"home_logo":home_logo.get("src","") if home_logo else "","away_logo":away_logo.get("src","") if away_logo else "","status":status_type}

# ===== VIDEO PROXY =====
@app.get("/api/proxy/m3u8")
def proxy_m3u8(url: str = Query(...)):
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        if r.status_code != 200: return Response(status_code=502, content="upstream error")
        lines = r.text.strip().split('\n'); rewritten = []
        for line in lines:
            if line.startswith('#') or not line.strip(): rewritten.append(line)
            else: rewritten.append("/api/proxy/seg?url=" + quote(line.strip(), safe=""))
        return Response(content='\n'.join(rewritten).encode('utf-8'), media_type="application/vnd.apple.mpegurl", headers={"Access-Control-Allow-Origin":"*","Cache-Control":"public, max-age=300"})
    except: return Response(status_code=502, content="proxy error")

@app.get("/api/proxy/seg")
def proxy_segment(url: str = Query(...)):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        if r.status_code != 200: return Response(status_code=502, content="upstream error")
        data = r.content
        if len(data) > 188 and data[0:4] == b'\x89PNG' and data[188] == 0x47: data = data[188:]
        return Response(content=data, media_type="video/mp2t", headers={"Access-Control-Allow-Origin":"*","Cache-Control":"public, max-age=3600"})
    except: return Response(status_code=502, content="proxy error")

@app.get("/api/proxy/video")
def proxy_video(url: str = Query(...), request: Request = None):
    try:
        req_headers = dict(HEADERS)
        if request and request.headers.get("range"): req_headers["Range"] = request.headers["range"]
        r = requests.get(url, headers=req_headers, timeout=30, stream=True)
        resp_headers = {"Access-Control-Allow-Origin":"*","Accept-Ranges":"bytes","Content-Type":r.headers.get("Content-Type","video/mp4")}
        if "Content-Range" in r.headers: resp_headers["Content-Range"] = r.headers["Content-Range"]
        if "Content-Length" in r.headers: resp_headers["Content-Length"] = r.headers["Content-Length"]
        return StreamingResponse(r.iter_content(chunk_size=256*1024), status_code=r.status_code, headers=resp_headers)
    except: return Response(status_code=502, content="proxy error")

@app.get("/api/proxy/img")
def proxy_img(url: str = Query(...)):
    try:
        r = requests.get(url, headers={**HEADERS, "Referer": "https://dantri.com.vn/"}, timeout=10)
        if r.status_code != 200: return Response(status_code=502)
        ct = r.headers.get("Content-Type", "image/jpeg")
        return Response(content=r.content, media_type=ct, headers={"Cache-Control": "public, max-age=86400", "Access-Control-Allow-Origin": "*"})
    except: return Response(status_code=502)

# ===== XEMLAIBONGDA HIGHLIGHTS =====
def _scrape_xemlaibongda_page(page_path, limit=20):
    """
    Scrape video từ xemlaibongda.top - Simple & Reliable
    Dùng logic cũ đã test, không fetch từng trang (tránh timeout)
    """
    try:
        url = f"https://xemlaibongda.top/{page_path}" if page_path else "https://xemlaibongda.top/"
        r = requests.get(url, headers=HEADERS, timeout=15)
        if r.status_code != 200:
            return []
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "lxml")
        videos = []
        seen = set()
        
        for a in soup.find_all("a", href=True):
            href = a.get("href", "")
            if "/video/" not in href and "/xem-lai/" not in href:
                continue
            
            if not href.startswith("http"):
                href = "https://xemlaibongda.top" + href
            
            # Bỏ query params
            clean_href = href.split("?")[0].split("#")[0]
            if clean_href in seen:
                continue
            seen.add(clean_href)
            
            # ===== Lấy THUMBNAIL =====
            img_src = ""
            img = a.find("img")
            if not img and a.parent:
                img = a.parent.find("img")
            if not img:
                p = a.parent
                for _ in range(4):
                    if p and p.find("img"):
                        img = p.find("img")
                        break
                    p = p.parent if p else None
            
            if img:
                img_src = (img.get("data-src", "") or img.get("src", "") or 
                          img.get("data-lazy", "") or img.get("data-original", ""))
                if img_src.startswith("//"):
                    img_src = "https:" + img_src
                elif img_src.startswith("/"):
                    img_src = "https://xemlaibongda.top" + img_src
            
            # ===== Lấy TITLE =====
            title = ""
            # Thử attribute
            for attr in ["title", "aria-label"]:
                val = a.get(attr, "")
                if val and len(val) >= 5:
                    title = val
                    break
            
            # Thử các selector
            if not title:
                for selector in ["h3", "h2", "h4", ".title", ".video-title", "strong"]:
                    try:
                        el = a.select_one(selector)
                        if el:
                            t = el.get_text(strip=True)
                            if len(t) >= 5:
                                title = t
                                break
                    except:
                        pass
            
            # Thử text content
            if not title:
                text = a.get_text(strip=True)
                if text and len(text) >= 5:
                    title = text[:100]
            
            # Fallback: tạo title từ slug
            if not title or len(title) < 3:
                slug = clean_href.split("/video/")[-1].rstrip("/").split("/xem-lai/")[-1].rstrip("/")
                title = slug.replace("-", " ").replace("_", " ").title()
                title = re.sub(r'\d{4}-\d{2}-\d{2}', '', title).strip()
            
            if not title or len(title) < 3:
                continue
            
            # Fallback thumbnail từ slug
            if not img_src:
                slug = clean_href.split("/video/")[-1].rstrip("/").split("/xem-lai/")[-1].rstrip("/")
                img_src = f"https://xemlaibongda.top/uploads/thumb/{slug}.jpg"
            
            videos.append({
                "title": title[:100],
                "link": clean_href,
                "img": img_src,
                "source": "xemlaibongda"
            })
            
            if len(videos) >= limit:
                break
        
        return videos
    except Exception as e:
        print(f"[xemlaibongda] Error: {e}")
        return []

def _extract_img_src(a_tag):
    """Extract image URL từ thẻ <a> và parent elements"""
    img = a_tag.find("img")
    if not img and a_tag.parent: 
        img = a_tag.parent.find("img")
    if not img:
        p = a_tag.parent
        for _ in range(5):  # Tìm sâu hơn
            if p and p.find("img"): 
                img = p.find("img")
                break
            p = p.parent if p else None
    
    if not img:
        return ""
    
    # Thử tất cả các attribute có thể chứa img URL
    attrs = ["data-src", "src", "data-lazy", "data-original", "data-srcset", "data-thumb", "data-image"]
    for attr in attrs:
        val = img.get(attr, "")
        if val:
            if attr == "data-srcset":
                val = val.split(",")[0].strip().split(" ")[0]
            break
    else:
        val = ""
    
    # Thử background-image từ style
    if not val:
        style = img.get("style", "") or img.get("data-bg", "")
        bg_match = re.search(r'url\(["\']?(.*?)["\']?\)', style)
        if bg_match:
            val = bg_match.group(1)
    
    # Normalize URL
    if val.startswith("//"): 
        val = "https:" + val
    elif val.startswith("/"): 
        val = "https://xemlaibongda.top" + val
    
    return val

def _extract_title(a_tag, href):
    """Extract title từ thẻ <a> và child/parent elements"""
    title = ""
    
    # 1. Thử các selector phổ biến cho title
    title_selectors = [
        "h3", "h2", "h4", "h5",
        ".title", ".post-title", ".entry-title", ".video-title", 
        ".card-title", ".item-title", ".news-title",
        "span.title", "strong", "b",
        ".name", ".caption"
    ]
    for tag in title_selectors:
        try:
            t = a_tag.select_one(tag) if hasattr(a_tag, 'select_one') else None
            if t: 
                title = t.get_text(" ", strip=True)
                if len(title) >= 3: 
                    return title
        except:
            pass
    
    # 2. Thử attribute của <a>
    for attr in ["title", "aria-label", "data-title"]:
        val = a_tag.get(attr, "")
        if val and len(val) >= 3:
            return val
    
    # 3. Thử alt text của img
    img = a_tag.find("img")
    if img:
        alt = img.get("alt", "")
        if alt and len(alt) >= 3:
            return alt
    
    # 4. Thử text content của <a> (loại bỏ quá dài)
    text = a_tag.get_text(" ", strip=True)
    if text and len(text) >= 3:
        # Lấy dòng đầu tiên nếu có nhiều dòng
        lines = [l.strip() for l in text.split("\n") if l.strip()]
        if lines:
            first_line = lines[0]
            if len(first_line) >= 3:
                return first_line[:100]
    
    # 5. Fallback: tạo title từ slug
    slug = href.split("/video/")[-1].rstrip("/").split("/xem-lai/")[-1].rstrip("/")
    title = slug.replace("-", " ").replace("_", " ")
    title = re.sub(r'\d{4}-\d{2}-\d{2}', '', title).strip()
    if title:
        return title.title()
    
    return ""

def scrape_xemlaibongda(): return _scrape_xemlaibongda_page("", 20)
def scrape_highlights_by_league(league_key):
    if league_key not in HL_LEAGUES: return []
    return _scrape_xemlaibongda_page(HL_LEAGUES[league_key]["path"], 20)
def scrape_all_league_highlights():
    results = {}
    def _fetch(key): return key, scrape_highlights_by_league(key)
    with ThreadPoolExecutor(8) as ex:
        futs = [ex.submit(_fetch, k) for k in HL_LEAGUES]
        for f in as_completed(futs, timeout=25):
            try:
                key, vids = f.result()
                if vids: results[key] = vids
            except: pass
    return results

def extract_xemlaibongda_video(url):
    try:
        r=requests.get(url, headers=HEADERS, timeout=15)
        if r.status_code!=200: return None
        r.encoding="utf-8"; soup=BeautifulSoup(r.text,"lxml"); video=soup.find("video")
        if video:
            src=video.get("src",""); poster=video.get("poster","")
            if not src:
                source=video.find("source")
                if source: src=source.get("src","")
            if src: return{"src":src,"poster":poster,"type":"hls" if".m3u8" in src else"video"}
        m3u8s=re.findall(r'(https?://[^\s"\'<>]+\.m3u8)',r.text)
        if m3u8s:
            og=soup.find("meta",property="og:image"); poster=og.get("content","") if og else ""
            return{"src":m3u8s[0],"poster":poster,"type":"hls"}
        return None
    except: return None

# ===== YOUTUBE SHORTS SCRAPING =====
def _yt_channel_shorts_requests(channel, count=15):
    try:
        url=f"https://www.youtube.com/@{channel}/shorts"
        r=requests.get(url, headers={**HEADERS,"Accept-Language":"vi,en;q=0.8"}, timeout=15)
        if r.status_code!=200: return []
        html=r.text; ids=[]; items=[]
        for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"',html):
            vid=m.group(1)
            if vid in ids: continue
            ids.append(vid)
            snip=html[max(0,m.start()-900):m.start()+1600]
            title=""
            mt=re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"',snip)
            if not mt: mt=re.search(r'"accessibilityText":"([^"]+)"',snip)
            if mt: title=html_lib.unescape(mt.group(1)).replace('\n',' ').strip()
            if not title: title="YouTube Short"
            items.append({"title":title,"link":f"https://www.youtube.com/watch?v={vid}","img":f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg","source":"yt","id":vid,"channel":channel})
            if len(items)>=count: break
        return items
    except: return []

def scrape_shorts():
    vids=[]
    with ThreadPoolExecutor(3) as ex:
        futs=[ex.submit(_yt_channel_shorts_requests,ch,24) for ch in ["baodantri7941","baosuckhoedoisongboyte","vtvnambo"]]
        for f in as_completed(futs):
            try:
                r=f.result()
                if r: vids.extend(r)
            except: pass
    merged=[]; seen=set()
    for v in vids:
        vid=v.get("id")
        if not vid or vid in seen: continue
        seen.add(vid); merged.append(v)
    for v in SHORTS_FALLBACK:
        vid=v.get("id")
        if not vid or vid in seen: continue
        seen.add(vid); merged.append(v)
    return merged[:60]

# ===== VTV NAM BO & WC SHORTS - using yt-dlp =====
from yt_scraper import get_vtvnambo_shorts, get_wc_related_shorts

@app.get("/api/shorts/vtvnamo")
def api_shorts_vtvnamo(count: int = Query(default=50, le=100)):
    items = get_vtvnambo_shorts(count)
    if not items:
        items = [v for v in SHORTS_FALLBACK if v.get("channel") == "vtvnambo"]
    nql = [v for v in items if v.get("id") == "nqlLH6chLRo"]
    rest = [v for v in items if v.get("id") != "nqlLH6chLRo"]
    items = nql + rest
    return JSONResponse(items)

@app.get("/api/shorts/wc")
def api_shorts_wc(count: int = Query(default=50, le=100)):
    items = get_wc_related_shorts(count)
    if not items:
        items = [v for v in SHORTS_FALLBACK if v.get("channel") == "vtvnambo"]
    nql = [v for v in items if v.get("id") == "nqlLH6chLRo"]
    rest = [v for v in items if v.get("id") != "nqlLH6chLRo"]
    items = nql + rest
    return JSONResponse(items)

# ===== LIVESCORE =====
@app.get("/api/livescore/live")
def api_livescore_live(): return JSONResponse({"html":_cached("ls_live",lambda:fetch_bongda_api("/api/fixtures/live"),ttl=_cache_ttl_live)})
@app.get("/api/livescore/incoming")
def api_livescore_incoming(): return JSONResponse({"html":_cached("ls_incoming",lambda:fetch_bongda_api("/api/fixtures/incoming"),ttl=_cache_ttl_live)})
@app.get("/api/livescore/today")
def api_livescore_today():
    today=datetime.now(VN_TZ).strftime("%Y-%m-%d");return JSONResponse({"html":_cached("ls_today",lambda:fetch_bongda_api(f"/api/fixtures/get-by-date?date={today}"),ttl=_cache_ttl)})
@app.get("/api/livescore/results")
def api_livescore_results():
    today=datetime.now(VN_TZ).strftime("%Y-%m-%d");return JSONResponse({"html":_cached("ls_results",lambda:fetch_bongda_api(f"/api/fixtures/get-by-date?date={today}&status=finished"),ttl=_cache_ttl)})
@app.get("/api/livescore/standings/{league}")
def api_livescore_standings(league:str):
    tid=LEAGUE_IDS.get(league,27110);return JSONResponse({"html":_cached(f"ls_bxh_{league}",lambda:fetch_bongda_api(f"/api/league-table/home?tournament_id={tid}&is_detail=True"),ttl=_cache_ttl)})
@app.get("/api/livescore/date/{date}")
def api_livescore_date(date:str):return JSONResponse({"html":fetch_bongda_api(f"/api/fixtures/get-by-date?date={date}")})
@app.get("/api/match/{event_id}/commentaries")
def api_match_commentaries(event_id:int):return JSONResponse({"html":fetch_bongda_api(f"/api/fixtures/commentaries?event_id={event_id}")})
@app.get("/api/match/{event_id}/stats")
def api_match_stats(event_id:int):return JSONResponse({"html":fetch_bongda_api(f"/api/event-standing/player-performance?event_id={event_id}")})

from match_detail_v2 import fetch_match_detail, fetch_match_detail_by_url

@app.get("/api/match/{event_id}/detail")
def api_match_detail(event_id: int, url: str = Query(default="")):
    try:
        if url: data = fetch_match_detail_by_url(url)
        else: data = fetch_match_detail(event_id)
        return JSONResponse(data)
    except Exception as e: return JSONResponse({"event_id": event_id, "found": False, "error": str(e)})

@app.get("/api/livescore/featured")
def api_livescore_featured():
    def _f():
        sources=[("/api/fixtures/live","live"),("/api/fixtures/get-by-date?date="+datetime.now(VN_TZ).strftime("%Y-%m-%d"),"today"),("/api/fixtures/incoming","upcoming")]
        for endpoint, stype in sources:
            html=fetch_bongda_api(endpoint)
            if not html or len(html)<100:continue
            soup=BeautifulSoup(html,"lxml");all_matches=[]
            for li in soup.select("li.match-detail"):
                match=_parse_match_from_li(li, stype)
                if not match or not match["event_id"]:continue
                if stype=="today" and "KT" in match.get("minute",""):continue
                all_matches.append(match)
            if not all_matches:continue
            for pl in PRIORITY_LEAGUES:
                for match in all_matches:
                    if pl in match["league"]:return match
            return all_matches[0]
        return None
    return JSONResponse(_cached("ls_featured",_f,ttl=30))

@app.get("/api/shorts")
def api_shorts(channel: str = Query(default="")):
    if channel == "vtvnambo": return api_shorts_vtvnamo()
    if channel == "wc": return api_shorts_wc()
    return JSONResponse(_cached("yt_shorts_v3",scrape_shorts,ttl=_cache_ttl_yt))

@app.get("/api/short-stats")
def api_short_stats(ids:str=Query(default="")):
    arr=[x for x in ids.split(",") if x]
    with _short_lock:
        db=_load_short_db();out={}
        for vid in arr:
            st=db.get(vid) or _short_default()
            out[vid]={"views":int(st.get("views",0)),"likes":int(st.get("likes",0)),"shares":int(st.get("shares",0)),"comments":st.get("comments",[])[:80]}
    return JSONResponse({"stats":out})

@app.post("/api/short-action")
async def api_short_action(request:Request):
    try: body=await request.json()
    except: body={}
    vid=str(body.get("id","")).strip(); action=str(body.get("action","")).strip(); txt=str(body.get("text","")).strip()
    if not vid: return JSONResponse({"error":"missing id"},status_code=400)
    with _short_lock:
        db=_load_short_db(); st=db.get(vid) or _short_default()
        if action=="view": st["views"]=int(st.get("views",0))+1
        elif action=="like": st["likes"]=int(st.get("likes",0))+1
        elif action=="share": st["shares"]=int(st.get("shares",0))+1
        elif action=="comment" and txt:
            comments=st.get("comments",[])
            comments.insert(0,{"text":txt[:180],"ts":int(time.time())})
            st["comments"]=comments[:80]
        st["updated"]=int(time.time()); db[vid]=st; _save_short_db(db)
        out={"views":int(st.get("views",0)),"likes":int(st.get("likes",0)),"shares":int(st.get("shares",0)),"comments":st.get("comments",[])[:80]}
    return JSONResponse({"stats":out})

@app.get("/api/highlights")
def api_highlights(): return JSONResponse(_cached("xemlaibongda_hl",scrape_xemlaibongda,ttl=_cache_ttl))
@app.get("/api/highlights/leagues")
def api_highlights_leagues(): return JSONResponse(_cached("hl_leagues",scrape_all_league_highlights,ttl=_cache_ttl))
@app.get("/api/highlights/{league}")
def api_highlights_league(league:str):
    if league not in HL_LEAGUES: return JSONResponse({"error":"league not found"})
    return JSONResponse(_cached(f"hl_{league}",lambda:scrape_highlights_by_league(league),ttl=_cache_ttl))

@app.get("/api/video_url")
def api_video_url(url:str=Query(...)):
    if "youtube.com" in url or "youtu.be" in url:
        m=re.search(r'(?:v=|shorts/|youtu\.be/)([a-zA-Z0-9_-]{11})',url)
        if m: vid=m.group(1); return JSONResponse({"src":f"https://www.youtube.com/embed/{vid}?autoplay=1&rel=0&enablejsapi=1","poster":f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg","type":"youtube"})
    if "xemlaibongda.top" in url:
        v=extract_xemlaibongda_video(url)
        if v:
            if v["type"]=="hls": v["src"]="/api/proxy/m3u8?url="+quote(v["src"],safe="")
            return JSONResponse(v)
    return JSONResponse({"error":"not found"})

# ===== WORLD CUP 2026 API =====
# Rate limiting cho WC API
_wc_request_times = []
_wc_rate_limit_lock = threading.Lock()
_WC_RATE_LIMIT = 10  # Max 10 requests per minute

def _wc_rate_limit():
    """Kiểm tra rate limit cho WC API"""
    global _wc_request_times
    with _wc_rate_limit_lock:
        now = time.time()
        # Xóa các request cũ hơn 60 giây
        _wc_request_times = [t for t in _wc_request_times if now - t < 60]
        if len(_wc_request_times) >= _WC_RATE_LIMIT:
            return False
        _wc_request_times.append(now)
        return True

@app.get("/api/wc2026")
def api_wc2026():
    """Trả về tất cả dữ liệu World Cup 2026"""
    return JSONResponse(_cached("wc2026", get_wc2026_all, ttl=_cache_ttl))

@app.get("/api/wc2026/{tab}")
def api_wc2026_tab(tab: str):
    """Trả về từng tab của World Cup"""
    valid_tabs = ["news", "fixtures", "standings", "stats", "highlights"]
    if tab not in valid_tabs:
        return JSONResponse({"error": "invalid tab"}, status_code=400)
    
    def _fetch_tab():
        if tab == "highlights":
            return scrape_highlights_by_league("world-cup")
        elif tab == "news":
            return scrape_wc_news()
        elif tab == "fixtures":
            return scrape_fixtures()
        elif tab == "standings":
            return scrape_standings()
        elif tab == "stats":
            return scrape_stats()
        return []
    
    return JSONResponse(_cached(f"wc2026_{tab}", _fetch_tab, ttl=_cache_ttl))

# Note: WC functions (scrape_wc_news, scrape_fixtures, scrape_stats, scrape_standings) 
# are imported from wc2026_scraper.py at the top of this file

@app.get("/api/video_url")
def api_video_url(url:str=Query(...)):
    if "youtube.com" in url or "youtu.be" in url:
        m=re.search(r'(?:v=|shorts/|youtu\.be/)([a-zA-Z0-9_-]{11})',url)
        if m: vid=m.group(1); return JSONResponse({"src":f"https://www.youtube.com/embed/{vid}?autoplay=1&rel=0&enablejsapi=1","poster":f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg","type":"youtube"})
    if "xemlaibongda.top" in url:
        v=extract_xemlaibongda_video(url)
        if v:
            if v["type"]=="hls": v["src"]="/api/proxy/m3u8?url="+quote(v["src"],safe="")
            return JSONResponse(v)
    return JSONResponse({"error":"not found"})

@app.get("/api/bdp_videos")
def api_bdp_videos():
    def _f():
        try:
            soup=_get(f"{BASE_BDP}/video"); arts=[]; seen=set()
            for a in soup.find_all("a",href=True):
                href=a.get("href","")
                if"/video/" not in href or href in("/video/","/video/ban-thang-dep","/video/highlight"):continue
                if not href.startswith("http"): href=BASE_BDP+href
                if href in seen: continue
                title=re.sub(r'^\d{2}:\d{2}','',a.get_text(strip=True)).strip()
                if not title or len(title)<5: continue
                img_tag=a.find("img") or(a.parent.find("img") if a.parent else None)
                img=(img_tag.get("data-src") or img_tag.get("src","")) if img_tag else ""
                seen.add(href); arts.append({"title":title,"link":href,"img":img,"source":"bdp"})
            return arts[:20]
        except: return []
    return JSONResponse(_cached("bdp_videos",_f))

# ===== NEWS =====
VNE_CATS={"thoi-su":("https://vnexpress.net/thoi-su","Thời Sự"),"the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),"kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),"the-thao":("https://vnexpress.net/the-thao","Thể Thao"),"giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),"suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe"),"phap-luat":("https://vnexpress.net/phap-luat","Pháp Luật"),"giao-duc":("https://vnexpress.net/giao-duc","Giáo Dục"),"du-lich":("https://vnexpress.net/du-lich","Du Lịch"),"doi-song":("https://vnexpress.net/doi-song","Đời Sống")}

def scrape_vne(cat_url):
    try:
        soup=_get(cat_url); arts=[]
        for it in soup.select("article.item-news")[:15]:
            a=it.select_one("h2.title-news a") or it.select_one("h3.title-news a")
            if not a: continue
            t=a.get("title","") or a.get_text(strip=True); lk=a.get("href","")
            if not t or not lk: continue
            im=it.find("img"); img=(im.get("data-src") or im.get("src","")) if im else ""
            if img and 'blank' in img:
                src=it.find("source")
                if src: img=src.get("srcset","").split(",")[0].strip().split(" ")[0]
            arts.append({"title":t,"link":lk,"img":img,"source":"vne"})
        return arts
    except: return []

def scrape_genk_ai():
    try:
        r=requests.get("https://genk.vn/ai.chn",headers=HEADERS,timeout=15)
        if r.status_code!=200: return []
        r.encoding="utf-8"; soup=BeautifulSoup(r.text,"lxml"); articles=[]; seen=set()
        for a in soup.find_all("a",href=True):
            href=a.get("href","")
            if not href.endswith(".chn") or href=="/ai.chn": continue
            if href.startswith("/"): href="https://genk.vn"+href
            if href in seen or "genk.vn" not in href: continue
            title=a.get("title","") or a.get_text(strip=True)
            if not title or len(title)<20: continue
            container=a.parent; img_src=""
            for _ in range(6):
                if container is None: break
                for img in container.find_all("img"):
                    s=img.get("data-src","") or img.get("src","")
                    if s and "mediacdn" in s and "avatar" not in s and "logo" not in s: img_src=s; break
                if img_src: break
                container=container.parent
            seen.add(href)
            if not img_src:
                try:
                    og_r=requests.get(href,headers=HEADERS,timeout=8); og_r.encoding="utf-8"
                    og_soup=BeautifulSoup(og_r.text,"lxml"); og_tag=og_soup.find("meta",property="og:image")
                    if og_tag: img_src=og_tag.get("content","")
                except: pass
            articles.append({"title":title,"link":href,"img":img_src,"source":"genk"})
            if len(articles)>=30: break
        return articles
    except: return []

@app.get("/api/homepage")
def api_homepage():
    def _f():
        articles=[]
        with ThreadPoolExecutor(12) as ex:
            futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri","phap-luat","giao-duc","du-lich","doi-song"]}
            for f in as_completed(futs):
                try:
                    for a in f.result(): a["group"]=futs[f]; articles.append(a)
                except: pass
        return articles
    return JSONResponse(_cached("homepage",_f))

@app.get("/api/category/{cat_id}")
def api_category(cat_id:str):
    def _f():
        if cat_id=="cong-nghe": return scrape_genk_ai()
        if cat_id in VNE_CATS:
            arts=scrape_vne(VNE_CATS[cat_id][0])
            [a.update({"group":VNE_CATS[cat_id][1]}) for a in arts]
            return arts
        return []
    return JSONResponse(_cached(f"cat_{cat_id}",_f))

@app.get("/api/categories")
def api_categories():
    cats=[{"id":"cong-nghe","name":"Công Nghệ","source":"genk"}]
    for k,(u,n) in VNE_CATS.items(): cats.append({"id":k,"name":n,"source":"vne"})
    return JSONResponse(cats)

@app.get("/api/article")
def api_article(url:str=Query(...)):
    return JSONResponse({"error":"not supported"})

@app.get("/api/storage_status")
def api_storage_status():
    return JSONResponse({"persistent":os.path.isdir("/data")})

@app.get("/api/hot_topics")
def api_hot_topics():
    return JSONResponse({"topics":[]})

@app.get("/", response_class=HTMLResponse)
async def root():
    return HTMLResponse("<h1>VNEWS</h1><p>Running</p>")
# v14 rebuild 2026-06-18T12:21:41.503230