Upload main.py

main.py

@@ -13,7 +13,7 @@ from bs4 import BeautifulSoup
 
 app = FastAPI()
 
-# ===== VTV CHANNELS API (VTV1-VTV10 + VTVPrime) =====
+# ===== VTV CHANNELS API =====
 from vtv_api import router as vtv_router
 app.include_router(vtv_router)
 
@@ -27,12 +27,9 @@ _cache_ttl_live = 60
 _cache_ttl_yt = 1800
 
 # ===== VTV NAM BỘ SHORTS FALLBACK =====
-# These are ALWAYS available even if all scraping methods fail
 SHORTS_FALLBACK = [
-    # VTV Nam Bộ shorts (verified working)
     {"id":"nqlLH6chLRo","title":"Tin nóng VTV Nam Bộ | #shorts","channel":"vtvnambo"},
     {"id":"E7Kq0v3hG6w","title":"VTV Nam Bộ - Tin tức miền Nam | #shorts","channel":"vtvnambo"},
-    # Dân trí shorts
     {"id":"Lu_iCQ5YwNM","title":"Công an lập hồ sơ xử lý người phụ nữ chửi bới, tát nam tài xế ô tô ở Hà Nội","channel":"baodantri7941"},
     {"id":"CwWvijF8BOA","title":"Chú rể Ninh Bình bật khóc nhận món quà bí mật người cha quá cố gửi 26 năm trước","channel":"baodantri7941"},
     {"id":"tvPewsc2ph4","title":"Tính năng ẩn trên iPhone giúp giảm mỏi mắt","channel":"baodantri7941"},
@@ -48,8 +45,7 @@ SHORTS_FALLBACK = [
     {"id":"bcpgRoxbLPw","title":"Giông lốc quật bay mái tôn ở TP.HCM","channel":"baodantri7941"},
     {"id":"ZIIC5osy544","title":"Bé trai Trung Quốc rơi từ tầng 11 vẫn sống sót kỳ diệu","channel":"baodantri7941"},
     {"id":"uTMJ49NQpyc","title":"Sau lớp mascot 40kg: Câu chuyện mưu sinh của người trẻ ở TPHCM","channel":"baodantri7941"},
-    # SKĐS shorts
-    {"id":"7Pd6vZ2Lz1M","title":"Hành động ấm lòng của người đàn ông tham gia tìm kiếm 5 học sinh tử vong ở sông Lô","channel":"baosuckhoedoisongboyte"},
+    {"id":"7Pd6vZ2Lz1M","title":"Hành động ấm lòng của người đàn ông tìm kiếm 5 học sinh tử vong ở sông Lô","channel":"baosuckhoedoisongboyte"},
     {"id":"SlHLt_ZyPiE","title":"Xử phạt người đàn ông xóa số điện thoại cứu hộ trên cao tốc Bắc - Nam","channel":"baosuckhoedoisongboyte"},
     {"id":"IUOprcJyYr4","title":"Phụ nữ táo bón có phải do lười ăn rau?","channel":"baosuckhoedoisongboyte"},
     {"id":"YY8ojFNE-AU","title":"Quái xế tự quay clip nẹt pô, đánh võng đăng TikTok bị xử lý","channel":"baosuckhoedoisongboyte"},
@@ -72,14 +68,12 @@ for _v in SHORTS_FALLBACK:
 
 SHORT_STATS_FILE = "/data/short_stats.json" if os.path.isdir("/data") else "/app/short_stats.json"
 _short_lock = threading.Lock()
-
 def _load_short_db():
     try:
         if os.path.exists(SHORT_STATS_FILE):
             with open(SHORT_STATS_FILE,"r",encoding="utf-8") as f: return json.load(f)
     except: pass
     return {}
-
 def _save_short_db(db):
     try:
         os.makedirs(os.path.dirname(SHORT_STATS_FILE), exist_ok=True)
@@ -87,18 +81,14 @@ def _save_short_db(db):
         with open(tmp,"w",encoding="utf-8") as f: json.dump(db, f, ensure_ascii=False)
         os.replace(tmp, SHORT_STATS_FILE)
     except: pass
-
 def _short_default(): return {"views":0,"likes":0,"shares":0,"comments":[]}
-
 WALL_FILE = "/data/wall_posts.json" if os.path.isdir("/data") else "/app/wall_posts.json"
-
 def _load_wall():
     try:
         if os.path.exists(WALL_FILE):
             with open(WALL_FILE,"r",encoding="utf-8") as f: return json.load(f)
     except: pass
     return []
-
 def _save_wall(posts):
     try:
         os.makedirs(os.path.dirname(WALL_FILE), exist_ok=True)
@@ -106,7 +96,6 @@ def _save_wall(posts):
         with open(tmp,"w",encoding="utf-8") as f: json.dump(posts[:100], f, ensure_ascii=False)
         os.replace(tmp, WALL_FILE)
     except: pass
-
 PRIORITY_LEAGUES = ["Ngoại Hạng Anh","FA Cup","Champions League","LaLiga","Copa del Rey","Serie A","Bundesliga","Ligue 1","V-League"]
 LEAGUE_IDS = {"nha":27110,"laliga":27233,"seriea":27044,"bundesliga":26891,"ligue1":27212}
 HL_LEAGUES = {
@@ -119,61 +108,41 @@ HL_LEAGUES = {
     "europa-league":{"path":"cup-chau-au/uefa-europa-league","name":"Europa League","emoji":"🟠"},
     "world-cup":{"path":"the-gioi/world-cup","name":"World Cup 2026","emoji":"🌍"},
 }
-
 def _cached(key, fn, ttl=None):
-    now = time.time(); t = ttl or _cache_ttl
-    if key in _cache and now - _cache[key]["t"] < t: return _cache[key]["d"]
-    try: data = fn()
-    except: data = _cache.get(key, {}).get("d", [])
-    _cache[key] = {"d": data, "t": now}
-    return data
-
+    now=time.time(); t=ttl or _cache_ttl
+    if key in _cache and now-_cache[key]["t"]<t: return _cache[key]["d"]
+    try: data=fn()
+    except: data=_cache.get(key,{}).get("d",[])
+    _cache[key]={"d":data,"t":now}; return data
 def _get(url, headers=None):
-    h = headers or HEADERS
-    r = requests.get(url, headers=h, timeout=15)
-    r.encoding = "utf-8"
-    return BeautifulSoup(r.text, "lxml")
-
+    h=headers or HEADERS; r=requests.get(url, headers=h, timeout=15); r.encoding="utf-8"
+    return BeautifulSoup(r.text,"lxml")
 def fetch_bongda_api(endpoint):
     try:
-        r = requests.get(f"https://bongda.com.vn{endpoint}", headers=BONGDA_HEADERS, timeout=10)
-        if r.status_code == 200:
-            data = r.json()
-            if data.get("status") == "success": return data.get("html", "")
+        r=requests.get(f"https://bongda.com.vn{endpoint}", headers=BONGDA_HEADERS, timeout=10)
+        if r.status_code==200:
+            data=r.json()
+            if data.get("status")=="success": return data.get("html","")
         return ""
     except: return ""
 
 def _parse_match_from_li(li, status_type="live"):
-    match_div = li.select_one("div.match")
+    match_div=li.select_one("div.match")
     if not match_div: return None
-    home_el = match_div.select_one(".home-team .name")
-    away_el = match_div.select_one(".away-team .name")
+    home_el=match_div.select_one(".home-team .name"); away_el=match_div.select_one(".away-team .name")
     if not home_el or not away_el: return None
-    status_el = match_div.select_one(".status a")
-    league_el = li.find_previous("strong")
-    time_el = match_div.select_one(".match-time")
-    home_logo = match_div.select_one(".home-team .logo img")
-    away_logo = match_div.select_one(".away-team .logo img")
-    event_id = ""
+    status_el=match_div.select_one(".status a"); league_el=li.find_previous("strong"); time_el=match_div.select_one(".match-time")
+    home_logo=match_div.select_one(".home-team .logo img"); away_logo=match_div.select_one(".away-team .logo img")
+    event_id=""
     if status_el:
-        href = status_el.get("href", "")
-        m = re.search(r'/tran-dau/(\d+)/', href)
-        if m: event_id = m.group(1)
-    spans = status_el.find_all("span") if status_el else []
-    score = ""; minute = ""
-    if len(spans) >= 3: score = f"{spans[0].get_text(strip=True)} - {spans[2].get_text(strip=True)}"
-    if len(spans) >= 4: minute = spans[3].get_text(strip=True)
-    if not score and status_el and status_el.select_one(".vs"): score = "VS"
-    league = league_el.get_text(strip=True) if league_el else ""
-    return {
-        "home": home_el.get_text(strip=True), "away": away_el.get_text(strip=True),
-        "score": score or "VS", "minute": minute, "league": league,
-        "time": time_el.get_text(strip=True) if time_el else "",
-        "event_id": event_id,
-        "home_logo": home_logo.get("src","") if home_logo else "",
-        "away_logo": away_logo.get("src","") if away_logo else "",
-        "status": status_type
-    }
+        href=status_el.get("href",""); m=re.search(r'/tran-dau/(\d+)/',href)
+        if m: event_id=m.group(1)
+    spans=status_el.find_all("span") if status_el else []; score=""; minute=""
+    if len(spans)>=3: score=f"{spans[0].get_text(strip=True)} - {spans[2].get_text(strip=True)}"
+    if len(spans)>=4: minute=spans[3].get_text(strip=True)
+    if not score and status_el and status_el.select_one(".vs"): score="VS"
+    league=league_el.get_text(strip=True) if league_el else ""
+    return {"home":home_el.get_text(strip=True),"away":away_el.get_text(strip=True),"score":score or"VS","minute":minute,"league":league,"time":time_el.get_text(strip=True) if time_el else "","event_id":event_id,"home_logo":home_logo.get("src","") if home_logo else "","away_logo":away_logo.get("src","") if away_logo else "","status":status_type}
 
 # ===== VIDEO PROXY =====
 @app.get("/api/proxy/m3u8")
@@ -185,7 +154,7 @@ def proxy_m3u8(url: str = Query(...)):
         for line in lines:
             if line.startswith('#') or not line.strip(): rewritten.append(line)
             else: rewritten.append("/api/proxy/seg?url=" + quote(line.strip(), safe=""))
-        return Response(content='\n'.join(rewritten).encode('utf-8'), media_type="application/vnd.apple.mpegurl", headers={"Access-Control-Allow-Origin":"*","Cache-Control":"public, max-age:300"})
+        return Response(content='\n'.join(rewritten).encode('utf-8'), media_type="application/vnd.apple.mpegurl", headers={"Access-Control-Allow-Origin":"*","Cache-Control":"public, max-age=300"})
     except: return Response(status_code=502, content="proxy error")
 
 @app.get("/api/proxy/seg")
@@ -195,7 +164,7 @@ def proxy_segment(url: str = Query(...)):
         if r.status_code != 200: return Response(status_code=502, content="upstream error")
         data = r.content
         if len(data) > 188 and data[0:4] == b'\x89PNG' and data[188] == 0x47: data = data[188:]
-        return Response(content=data, media_type="video/mp2t", headers={"Access-Control-Allow-Origin":"*","Cache-Control":"public, max-age:3600"})
+        return Response(content=data, media_type="video/mp2t", headers={"Access-Control-Allow-Origin":"*","Cache-Control":"public, max-age=3600"})
     except: return Response(status_code=502, content="proxy error")
 
 @app.get("/api/proxy/video")
@@ -216,7 +185,7 @@ def proxy_img(url: str = Query(...)):
         r = requests.get(url, headers={**HEADERS, "Referer": "https://dantri.com.vn/"}, timeout=10)
         if r.status_code != 200: return Response(status_code=502)
         ct = r.headers.get("Content-Type", "image/jpeg")
-        return Response(content=r.content, media_type=ct, headers={"Cache-Control": "public, max-age:86400", "Access-Control-Allow-Origin": "*"})
+        return Response(content=r.content, media_type=ct, headers={"Cache-Control": "public, max-age=86400", "Access-Control-Allow-Origin": "*"})
     except: return Response(status_code=502)
 
 # ===== XEMLAIBONGDA HIGHLIGHTS =====
@@ -283,7 +252,6 @@ def scrape_xemlaibongda(): return _scrape_xemlaibongda_page("", 20)
 def scrape_highlights_by_league(league_key):
     if league_key not in HL_LEAGUES: return []
     return _scrape_xemlaibongda_page(HL_LEAGUES[league_key]["path"], 20)
-
 def scrape_all_league_highlights():
     results = {}
     def _fetch(key): return key, scrape_highlights_by_league(key)
@@ -298,72 +266,70 @@ def scrape_all_league_highlights():
 
 def extract_xemlaibongda_video(url):
     try:
-        r = requests.get(url, headers=HEADERS, timeout=15)
-        if r.status_code != 200: return None
-        r.encoding = "utf-8"; soup = BeautifulSoup(r.text, "lxml")
-        video = soup.find("video")
+        r=requests.get(url, headers=HEADERS, timeout=15)
+        if r.status_code!=200: return None
+        r.encoding="utf-8"; soup=BeautifulSoup(r.text,"lxml"); video=soup.find("video")
         if video:
-            src = video.get("src",""); poster = video.get("poster","")
+            src=video.get("src",""); poster=video.get("poster","")
             if not src:
-                source = video.find("source")
-                if source: src = source.get("src","")
-            if src: return {"src":src,"poster":poster,"type":"hls" if ".m3u8" in src else "video"}
-        m3u8s = re.findall(r'(https?://[^\s"\'<>]+\.m3u8)', r.text)
+                source=video.find("source")
+                if source: src=source.get("src","")
+            if src: return{"src":src,"poster":poster,"type":"hls" if".m3u8" in src else"video"}
+        m3u8s=re.findall(r'(https?://[^\s"\'<>]+\.m3u8)',r.text)
         if m3u8s:
-            og = soup.find("meta", property="og:image")
-            poster = og.get("content","") if og else ""
-            return {"src":m3u8s[0],"poster":poster,"type":"hls"}
+            og=soup.find("meta",property="og:image"); poster=og.get("content","") if og else ""
+            return{"src":m3u8s[0],"poster":poster,"type":"hls"}
         return None
     except: return None
 
-# ===== YOUTUBE SHORTS SCRAPING (legacy) =====
+# ===== YOUTUBE SHORTS SCRAPING (legacy requests fallback) =====
 def _yt_channel_shorts_requests(channel, count=15):
     try:
-        url = f"https://www.youtube.com/@{channel}/shorts"
-        r = requests.get(url, headers={**HEADERS, "Accept-Language":"vi,en;q=0.8"}, timeout=15)
-        if r.status_code != 200: return []
-        html = r.text; ids = []; items = []
-        for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"', html):
-            vid = m.group(1)
+        url=f"https://www.youtube.com/@{channel}/shorts"
+        r=requests.get(url, headers={**HEADERS,"Accept-Language":"vi,en;q=0.8"}, timeout=15)
+        if r.status_code!=200: return []
+        html=r.text; ids=[]; items=[]
+        for m in re.finditer(r'"videoId":"([A-Za-z0-9_-]{11})"',html):
+            vid=m.group(1)
             if vid in ids: continue
             ids.append(vid)
-            snip = html[max(0, m.start()-900):m.start()+1600]
-            title = ""
-            mt = re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"', snip)
-            if not mt: mt = re.search(r'"accessibilityText":"([^"]+)"', snip)
-            if mt: title = html_lib.unescape(mt.group(1)).replace('\n',' ').strip()
-            if not title: title = "YouTube Short"
+            snip=html[max(0,m.start()-900):m.start()+1600]
+            title=""
+            mt=re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"',snip)
+            if not mt: mt=re.search(r'"accessibilityText":"([^"]+)"',snip)
+            if mt: title=html_lib.unescape(mt.group(1)).replace('\n',' ').strip()
+            if not title: title="YouTube Short"
             items.append({"title":title,"link":f"https://www.youtube.com/watch?v={vid}","img":f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg","source":"yt","id":vid,"channel":channel})
-            if len(items) >= count: break
+            if len(items)>=count: break
         return items
     except: return []
 
 def scrape_shorts():
-    vids = []
+    vids=[]
     with ThreadPoolExecutor(3) as ex:
-        futs = [ex.submit(_yt_channel_shorts_requests, ch, 24) for ch in ["baodantri7941","baosuckhoedoisongboyte","vtvnambo"]]
+        futs=[ex.submit(_yt_channel_shorts_requests,ch,24) for ch in ["baodantri7941","baosuckhoedoisongboyte","vtvnambo"]]
         for f in as_completed(futs):
             try:
-                r = f.result()
+                r=f.result()
                 if r: vids.extend(r)
             except: pass
-    merged = []; seen = set()
+    merged=[]; seen=set()
     for v in vids:
-        vid = v.get("id")
+        vid=v.get("id")
         if not vid or vid in seen: continue
         seen.add(vid); merged.append(v)
     for v in SHORTS_FALLBACK:
-        vid = v.get("id")
+        vid=v.get("id")
         if not vid or vid in seen: continue
         seen.add(vid); merged.append(v)
     return merged[:60]
 
-# ===== VTV NAM BỘ & WC SHORTS - using Piped API + fallback =====
-from piped_client import get_vtvnambo_shorts, get_wc_related_shorts
+# ===== VTV NAM BỘ & WC SHORTS - using yt-dlp =====
+from yt_scraper import get_vtvnambo_shorts, get_wc_related_shorts
 
 @app.get("/api/shorts/vtvnamo")
 def api_shorts_vtvnamo(count: int = Query(default=50, le=100)):
-    """Get all shorts from VTV Nam Bộ channel (@vtvnambo)."""
+    """Get all shorts from VTV Nam Bộ channel (@vtvnambo) using yt-dlp."""
     items = get_vtvnambo_shorts(count)
     if not items:
         items = [v for v in SHORTS_FALLBACK if v.get("channel") == "vtvnambo"]
@@ -375,7 +341,7 @@ def api_shorts_vtvnamo(count: int = Query(default=50, le=100)):
 
 @app.get("/api/shorts/wc")
 def api_shorts_wc(count: int = Query(default=50, le=100)):
-    """Get World Cup / football related shorts."""
+    """Get World Cup / football related shorts using yt-dlp."""
     items = get_wc_related_shorts(count)
     if not items:
         items = [v for v in SHORTS_FALLBACK if v.get("channel") == "vtvnambo"]
@@ -530,26 +496,6 @@ def scrape_vne(cat_url):
         return arts
     except: return []
 
-def _scrape_dantri_homepage(cat_filter=None):
-    try:
-        soup=_get("https://dantri.com.vn/"); arts=[]; seen=set()
-        for a in soup.find_all("a",href=True):
-            href=a.get("href",""); title=a.get("title","") or a.get_text(strip=True)
-            if not title or len(title)<15 or "javascript:" in href: continue
-            if not href.startswith("http"): href="https://dantri.com.vn"+href
-            if href in seen or not href.endswith(".htm"): continue
-            if cat_filter and f"/{cat_filter}/" not in href: continue
-            img_tag=a.find("img")
-            if not img_tag and a.parent: img_tag=a.parent.find("img")
-            img_src=""
-            if img_tag: img_src=img_tag.get("data-src","") or img_tag.get("src","")
-            if not img_src or "cdn" not in img_src: continue
-            proxied_img="/api/proxy/img?url="+quote(img_src,safe="")
-            seen.add(href); arts.append({"title":title,"link":href,"img":proxied_img,"source":"dantri"})
-            if len(arts)>=15: break
-        return arts
-    except: return []
-
 def scrape_genk_ai():
     try:
         r=requests.get("https://genk.vn/ai.chn",headers=HEADERS,timeout=15)
@@ -584,7 +530,7 @@ def scrape_genk_ai():
 
 @app.get("/api/homepage")
 def api_homepage():
-    def _f():
+    def _f()
         articles=[]
         with ThreadPoolExecutor(12) as ex:
             futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri","phap-luat","giao-duc","du-lich","doi-song"]}
@@ -614,8 +560,7 @@ def api_categories():
 
 @app.get("/api/article")
 def api_article(url:str=Query(...)):
-    data=None
-    return JSONResponse(data if data else{"error":"not supported"})
+    return JSONResponse({"error":"not supported"})
 
 @app.get("/api/storage_status")
 def api_storage_status():