Add per-user cookies + proxy for YouTube access (UI fields + operator secrets)

A hosted Space can't read a visitor's browser cookies, so make access explicit and
self-serve:
- UI: "YouTube access" panel with a cookies box (raw cookies.txt or base64) and a proxy
URL box, used only for that run; throwaway-account warning included
- Fall back to operator-wide YT_COOKIES / YT_PROXY secrets when the UI fields are empty
- Thread proxy through download + comment-fetching; add player-client fallbacks
(default/android/tv/web_safari) for best-effort access without cookies
- Honest caveat in UI + README: free datacenter proxies (e.g. Webshare free tier) usually
do NOT bypass YouTube's block

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

README.md

@@ -46,8 +46,16 @@ on that topic.
   <https://huggingface.co/settings/tokens>.
 - **Free CPU tier:** Whisper runs on CPU, so transcription is slow — keep videos short
   (default cap ~20 min).
-- **YouTube may block the Space's IP.** If downloads/comments fail, add a Netscape-format
-  cookie file as a Space secret named `YT_COOKIES` (the file's contents).
+- **YouTube usually blocks the Space's datacenter IP.** Each visitor can supply their own
+  access in the **"YouTube access — cookies / proxy"** panel (used only for that run, then
+  deleted):
+  - **Cookies:** a `youtube.com` cookies.txt (Netscape format, raw or base64) exported
+    from a **throwaway** Google account.
+  - **Proxy:** a **residential** proxy URL. **Free *datacenter* proxies (e.g. Webshare's
+    free tier) usually do *not* bypass YouTube's block** and have tight bandwidth caps.
+  - An operator can set shared defaults via the `YT_COOKIES` / `YT_PROXY` Space secrets.
+  - A hosted Space **cannot** open or read a visitor's browser — auth must be supplied
+    explicitly.
 
 ## Local run
 

app.py

@@ -60,13 +60,14 @@ def _maybe_b64_decode(text: str) -> str | None:
     return decoded if _looks_like_netscape(decoded) else None
 
 
-def _cookiefile(workdir: str) -> str | None:
-    """Materialize the YT_COOKIES secret to a Netscape cookie file on disk.
+def _cookiefile(workdir: str, raw: str | None = None) -> str | None:
+    """Materialize cookies to a Netscape cookie file on disk; return its path or None.
 
-    Accepts either the raw cookies.txt contents (tabs preserved) or a base64 encoding of
-    them. A missing header line is added so yt-dlp accepts the file.
+    ``raw`` is the per-user UI value; if empty we fall back to the operator-wide
+    ``YT_COOKIES`` secret. Accepts either raw cookies.txt contents (tabs preserved) or a
+    base64 encoding of them. A missing header line is added so yt-dlp accepts the file.
     """
-    data = os.environ.get("YT_COOKIES")
+    data = (raw or "").strip() or os.environ.get("YT_COOKIES")
     if not data or not data.strip():
         return None
 
@@ -86,6 +87,12 @@ def _cookiefile(workdir: str) -> str | None:
     return path
 
 
+def _resolve_proxy(raw: str | None = None) -> str | None:
+    """Per-user proxy URL, falling back to the operator-wide YT_PROXY secret."""
+    proxy = (raw or "").strip() or os.environ.get("YT_PROXY", "").strip()
+    return proxy or None
+
+
 def _ranking_rows(scored: list[dict]) -> list[list]:
     rows = []
     for rank, v in enumerate(scored, start=1):
@@ -122,6 +129,7 @@ def _collect_keywords(primary_kw, secondary_kw) -> dict:
 
 def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
                  max_minutes, max_shots, primary_kw, secondary_kw,
+                 cookies_text, proxy_url,
                  progress=gr.Progress()):
     """Generator that yields (status_md, ranking_df, transcript, docx_file)."""
     log: list[str] = []
@@ -140,7 +148,15 @@ def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
     frames_dir = os.path.join(workdir, "frames")
     video_path = None
     try:
-        cookiefile = _cookiefile(workdir)
+        cookiefile = _cookiefile(workdir, cookies_text)
+        proxy = _resolve_proxy(proxy_url)
+        auth_bits = []
+        if cookiefile:
+            auth_bits.append("cookies")
+        if proxy:
+            auth_bits.append("proxy")
+        if auth_bits:
+            yield status("🔐 Using " + " + ".join(auth_bits) + " for YouTube access."), gr.update(), gr.update(), gr.update()
 
         # 1. Search ------------------------------------------------------------------
         progress(0.02, desc="Searching")
@@ -150,7 +166,7 @@ def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
 
         # 2. Sentiment ranking -------------------------------------------------------
         yield status("💬 Fetching comments and scoring sentiment…"), gr.update(), gr.update(), gr.update()
-        best, scored = sentiment_mod.rank_by_sentiment(videos, cookiefile, progress)
+        best, scored = sentiment_mod.rank_by_sentiment(videos, cookiefile, progress, proxy)
         ranking = gr.update(value=_ranking_rows(scored))
         yield (status(f"🏆 Picked **{best.get('title', best['video_id'])}** "
                       f"({best['positive_share'] * 100:.0f}% positive)."),
@@ -160,7 +176,7 @@ def run_pipeline(topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
         progress(0.25, desc="Downloading")
         yield status("⬇️ Downloading the chosen video…"), ranking, gr.update(), gr.update()
         video_path, duration = download_mod.download_video(
-            best["url"], workdir, cookiefile, int(max_minutes))
+            best["url"], workdir, cookiefile, int(max_minutes), proxy)
         wav = download_mod.extract_audio(video_path, workdir)
 
         # 4. Transcribe --------------------------------------------------------------
@@ -246,6 +262,28 @@ def build_ui():
                 vlm_model = gr.Dropdown(VLM_CHOICES, value=VLM_CHOICES[0],
                                         label="Vision model (captions)", allow_custom_value=True)
 
+        with gr.Accordion("YouTube access — cookies / proxy (often required)", open=False):
+            gr.Markdown(
+                "⚠️ **YouTube usually blocks the Space's datacenter IP.** To download, give "
+                "the Space **your own** access below — it is used only for your run and "
+                "deleted afterward.\n\n"
+                "- **Use a throwaway Google account, not your main one.** yt-dlp activity "
+                "can get an account rate-limited or flagged.\n"
+                "- **Cookies:** export a `youtube.com` cookies.txt (Netscape format) from a "
+                "logged-in throwaway account and paste it (raw or base64) below.\n"
+                "- **Proxy:** a **residential** proxy works; **free *datacenter* proxies "
+                "(e.g. Webshare's free tier) usually do NOT** get past YouTube's block and "
+                "have tight bandwidth caps.\n"
+                "- An operator can instead set Space secrets `YT_COOKIES` / `YT_PROXY` as "
+                "shared defaults."
+            )
+            cookies_text = gr.Textbox(
+                label="YouTube cookies (cookies.txt contents or base64)", lines=4,
+                placeholder="# Netscape HTTP Cookie File …  (or a base64 blob)")
+            proxy_url = gr.Textbox(
+                label="Proxy URL (optional)", type="password",
+                placeholder="http://user:pass@host:port")
+
         with gr.Accordion("SEO / AEO keywords (optional)", open=False):
             gr.Markdown(
                 "The **primary keyword** is used naturally ~3× in the body and placed in "
@@ -281,7 +319,8 @@ def build_ui():
         run_btn.click(
             run_pipeline,
             inputs=[topic, hf_token, llm_model, vlm_model, w_llm, w_whisper, lead,
-                    max_minutes, max_shots, primary_kw, secondary_kw],
+                    max_minutes, max_shots, primary_kw, secondary_kw,
+                    cookies_text, proxy_url],
             outputs=[status_md, ranking_df, transcript_box, docx_file],
         )
     return demo

pipeline/download.py

@@ -2,6 +2,10 @@
 
 Downloads a <=720p mp4 with yt-dlp, then uses ffmpeg to produce a 16 kHz mono wav.
 Enforces a duration cap up front so a long video can't stall the free-CPU Space.
+
+Both YouTube-facing calls accept an optional ``cookiefile`` and ``proxy`` (per-user or
+operator-wide), and fall back across a few yt-dlp player clients to improve the odds of
+getting past datacenter-IP blocking without cookies.
 """
 from __future__ import annotations
 
@@ -10,61 +14,81 @@ import subprocess
 
 from yt_dlp import YoutubeDL
 
+# Player clients tried in order. "android"/"tv" sometimes succeed where "web" is blocked.
+_PLAYER_CLIENTS = ["default", "android", "tv", "web_safari"]
+
 
 class DownloadError(RuntimeError):
     """Raised for user-actionable download failures (blocked IP, too long, etc.)."""
 
 
-def probe_duration(url: str, cookiefile: str | None = None) -> float:
-    """Return the video duration in seconds without downloading."""
-    opts = {"skip_download": True, "quiet": True, "no_warnings": True}
+def _base_opts(cookiefile: str | None, proxy: str | None, player_client: str) -> dict:
+    opts = {
+        "quiet": True,
+        "no_warnings": True,
+        "noplaylist": True,
+        "extractor_args": {"youtube": {"player_client": [player_client]}},
+    }
     if cookiefile:
         opts["cookiefile"] = cookiefile
-    try:
-        with YoutubeDL(opts) as ydl:
-            info = ydl.extract_info(url, download=False)
-    except Exception as exc:
-        raise DownloadError(_blocked_hint(exc)) from exc
+    if proxy:
+        opts["proxy"] = proxy
+    return opts
+
+
+def _extract_with_fallback(url: str, base: dict, cookiefile: str | None,
+                           proxy: str | None, download: bool):
+    """Run extract_info, retrying across player clients on failure.
+
+    Returns ``(info, ydl)`` from the first client that works; raises DownloadError with a
+    helpful hint if all clients fail.
+    """
+    last_exc = None
+    for client in _PLAYER_CLIENTS:
+        opts = {**_base_opts(cookiefile, proxy, client), **base}
+        try:
+            ydl = YoutubeDL(opts)
+            info = ydl.extract_info(url, download=download)
+            return info, ydl
+        except Exception as exc:  # try the next client
+            last_exc = exc
+            continue
+    raise DownloadError(_blocked_hint(last_exc, bool(cookiefile), bool(proxy)))
+
+
+def probe_duration(url: str, cookiefile: str | None = None,
+                   proxy: str | None = None) -> float:
+    """Return the video duration in seconds without downloading."""
+    info, _ = _extract_with_fallback(url, {"skip_download": True}, cookiefile, proxy,
+                                     download=False)
     return float(info.get("duration") or 0.0)
 
 
 def download_video(url: str, out_dir: str, cookiefile: str | None = None,
-                   max_minutes: int = 20) -> tuple[str, float]:
+                   max_minutes: int = 20, proxy: str | None = None) -> tuple[str, float]:
     """Download the video to ``out_dir``; return ``(mp4_path, duration_seconds)``."""
     os.makedirs(out_dir, exist_ok=True)
 
-    duration = probe_duration(url, cookiefile)
+    duration = probe_duration(url, cookiefile, proxy)
     if duration and duration > max_minutes * 60:
         raise DownloadError(
             f"Video is {duration / 60:.0f} min long, over the {max_minutes} min cap for "
             "this free-CPU Space. Pick a shorter video or raise the cap."
         )
 
-    out_tmpl = os.path.join(out_dir, "video.%(ext)s")
-    opts = {
-        "outtmpl": out_tmpl,
+    base = {
+        "outtmpl": os.path.join(out_dir, "video.%(ext)s"),
         "format": "bestvideo[height<=720][ext=mp4]+bestaudio[ext=m4a]/best[height<=720]/best",
         "merge_output_format": "mp4",
-        "quiet": True,
-        "no_warnings": True,
-        "noplaylist": True,
     }
-    if cookiefile:
-        opts["cookiefile"] = cookiefile
-
-    try:
-        with YoutubeDL(opts) as ydl:
-            info = ydl.extract_info(url, download=True)
-            path = ydl.prepare_filename(info)
-    except Exception as exc:
-        raise DownloadError(_blocked_hint(exc)) from exc
-
+    info, ydl = _extract_with_fallback(url, base, cookiefile, proxy, download=True)
+    path = ydl.prepare_filename(info)
     duration = float(info.get("duration") or duration or 0.0)
 
     # merge_output_format may have rewritten the extension to .mp4
     if not os.path.exists(path):
-        base, _ = os.path.splitext(path)
-        path = base + ".mp4"
+        base_path, _ = os.path.splitext(path)
+        path = base_path + ".mp4"
     if not os.path.exists(path):
         raise DownloadError("yt-dlp reported success but no output file was found.")
     return path, duration
@@ -86,13 +110,21 @@ def extract_audio(video_path: str, out_dir: str) -> str:
     return wav_path
 
 
-def _blocked_hint(exc: Exception) -> str:
-    msg = str(exc)
+def _blocked_hint(exc: Exception | None, had_cookies: bool, had_proxy: bool) -> str:
+    msg = str(exc) if exc else "unknown error"
     lowered = msg.lower()
     if any(k in lowered for k in ("sign in", "bot", "403", "429", "confirm you", "cookies")):
+        used = []
+        if had_cookies:
+            used.append("cookies")
+        if had_proxy:
+            used.append("proxy")
+        ctx = (f" Your {' and '.join(used)} did not get past it"
+               if used else " No cookies or proxy were provided")
         return (
-            "YouTube blocked this request (common from datacenter IPs like HF Spaces). "
-            "Add a Netscape cookie file as the Space secret YT_COOKIES and retry.\n"
+            "YouTube blocked this request (typical from datacenter IPs)." + ctx + ". "
+            "Provide throwaway-account cookies and/or a residential proxy and retry. "
+            "Note: free datacenter proxies usually do NOT bypass this block.\n"
             f"Original error: {msg[:300]}"
         )
     return f"Download failed: {msg[:400]}"

pipeline/sentiment.py

@@ -50,7 +50,7 @@ def _polarity(scores) -> float:
     return val
 
 
-def _fetch_comments(url: str, cookiefile: str | None) -> list[str]:
+def _fetch_comments(url: str, cookiefile: str | None, proxy: str | None) -> list[str]:
     opts = {
         "skip_download": True,
         "getcomments": True,
@@ -60,6 +60,8 @@ def _fetch_comments(url: str, cookiefile: str | None) -> list[str]:
     }
     if cookiefile:
         opts["cookiefile"] = cookiefile
+    if proxy:
+        opts["proxy"] = proxy
     with YoutubeDL(opts) as ydl:
         info = ydl.extract_info(url, download=False)
     comments = info.get("comments") or []
@@ -68,7 +70,7 @@ def _fetch_comments(url: str, cookiefile: str | None) -> list[str]:
 
 
 def rank_by_sentiment(videos: list[dict], cookiefile: str | None = None,
-                      progress=None) -> tuple[dict, list[dict]]:
+                      progress=None, proxy: str | None = None) -> tuple[dict, list[dict]]:
     """Score each video and return ``(best_video, scored)``.
 
     ``scored`` mirrors ``videos`` with added keys: ``positive_share`` (0..1),
@@ -85,7 +87,7 @@ def rank_by_sentiment(videos: list[dict], cookiefile: str | None = None,
             progress((i + 1) / len(videos), desc=f"Sentiment {i + 1}/{len(videos)}")
         item = dict(v)
         try:
-            comments = _fetch_comments(v["url"], cookiefile)
+            comments = _fetch_comments(v["url"], cookiefile, proxy)
             if comments:
                 results = clf(comments)  # list aligned with comments (each = list of labels)
                 pols = [_polarity(r) for r in results]