Fix shorts/video classification: tag is_short deterministically from Jina channel tabs (no YouTube HEAD), override /api/shorts

app_v2_entry.py

@@ -165,6 +165,75 @@ def _search_all(topic,limit=36):
         for s in srcs:
             if i<len(s) and s[i].get('url') and s[i]['url'] not in seen:seen.add(s[i]['url']);out.append(s[i])
     return out[:limit]
+
+# ===== YOUTUBE SHORTS — reliable short/video classification (override) =====
+# Datacenter IPs (HF Space) get an EMPTY feeds/videos.xml from YouTube and YouTube
+# HEAD requests time out, so main.py's HEAD-based _classify_short always defaulted to
+# is_short=True -> every Dân trí/SKĐS item showed as "Short" and the 1:1 toggle never
+# appeared. Fix: classify deterministically from the Jina-rendered channel tabs —
+# a link under /shorts/<id> is a Short, a link under /watch?v=<id> on the /videos tab
+# is a normal (horizontal) video. No request to youtube.com is needed.
+from main import (_yt_jina_page, _yt_rss_shorts, SHORTS_FALLBACK,
+                  _YT_CHANNEL_IDS, _cached, _cache_ttl_yt)
+
+def _yt_jina_shorts_classified(channel, count=24):
+    """Newest videos for a channel via Jina channel tabs, each tagged is_short.
+    /shorts tab -> Short (is_short=True). /videos tab -> normal video (is_short=False)
+    unless the rendered link itself is a /shorts/ URL."""
+    items=[];seen=set()
+    # (tab, default_is_short_for_watch_links)
+    for path, tab_short in (("videos", False), ("shorts", True)):
+        md=_yt_jina_page(channel, path)
+        if not md:continue
+        for m in re.finditer(r'\[([^\]\[]{8,160})\]\(https?://(?:www\.)?youtube\.com/(shorts/|watch\?v=)([A-Za-z0-9_-]{11})[^)]*\)', md):
+            title=html_lib.unescape(m.group(1)).strip().strip('"')
+            form=m.group(2); vid=m.group(3)
+            if vid in seen:continue
+            tl=title.lower()
+            if tl in ("home home","shorts shorts","image") or title.startswith("!") or title.startswith("Image"):continue
+            # A /shorts/ URL is always a Short; a /watch?v= link inherits the tab default.
+            is_short = True if form.startswith("shorts/") else tab_short
+            seen.add(vid)
+            items.append({"title":title,"link":f"https://www.youtube.com/watch?v={vid}",
+                          "img":f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg","source":"yt",
+                          "id":vid,"channel":channel,"is_short":is_short})
+            if len(items)>=count:break
+    return items
+
+def _scrape_shorts_v2():
+    """Stable Dân trí & SKĐS feed with RELIABLE is_short tags (Jina tabs), newest-first.
+    Falls back to RSS (tagged short) then the static list so the slider never empties."""
+    vids=[]
+    with ThreadPoolExecutor(2) as ex:
+        futs=[ex.submit(_yt_jina_shorts_classified,ch,24) for ch in ["baodantri7941","baosuckhoedoisongboyte"]]
+        for f in as_completed(futs):
+            try:
+                r=f.result()
+                if r:vids.extend(r)
+            except:pass
+    # If Jina returned nothing for a channel, fill from RSS (newest, tagged short by default).
+    have=set(v.get("channel") for v in vids)
+    for ch in ["baodantri7941","baosuckhoedoisongboyte"]:
+        if ch not in have:
+            for v in _yt_rss_shorts(ch,24):
+                v.setdefault("is_short",True);vids.append(v)
+    merged=[];seen=set()
+    for v in vids+[dict(x) for x in SHORTS_FALLBACK]:
+        vid=v.get("id")
+        if not vid or vid in seen:continue
+        # Static fallback items: infer from title (#shorts -> short, else normal video).
+        if "is_short" not in v:
+            t=(v.get("title","") or "").lower()
+            v["is_short"]= ("#shorts" in t) or ("short" in t)
+        seen.add(vid);merged.append(v)
+    return merged[:40]
+
+# Override main.py's /api/shorts so the slider uses the reliable classifier.
+app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/api/shorts' and 'GET' in getattr(r,'methods',set()))]
+@app.get('/api/shorts')
+def api_shorts_v2():
+    return JSONResponse(_cached("yt_shorts_v4",_scrape_shorts_v2,ttl=_cache_ttl_yt))
+
 # Override article endpoint
 app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/api/article' and 'GET' in getattr(r,'methods',set()))]
 _JINA_BOILER_STOP=['facebooktelegram','copy link','follow us','bài viết liên quan','tin liên quan',