deploy 91f0b54

backend/package-lock.json

@@ -14,6 +14,7 @@
         "@sentry/node": "^10.53.1",
         "@sentry/profiling-node": "^10.53.1",
         "better-sse": "^0.16.1",
+        "bgutils-js": "3.2.0",
         "bullmq": "^5.73.0",
         "cheerio": "^1.2.0",
         "compression": "^1.8.1",
@@ -29,6 +30,7 @@
         "helmet": "^8.2.0",
         "ioredis": "^5.10.1",
         "jiti": "^2.6.1",
+        "jsdom": "27.4.0",
         "lru-cache": "^11.3.6",
         "metascraper": "^5.50.3",
         "metascraper-author": "^5.50.1",
@@ -1442,6 +1444,11 @@
       }
     },
     "node_modules/@libsql/android-arm64": {
+      "name": "null",
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/null/-/null-2.0.0.tgz",
+      "integrity": "sha512-CVSXPeB7Af0Sku7w8N8EKkgcwvQ6wGNfs6bly419A/q6dpIwzaHIk+65+La2g80ml21uxVXQo8xReTMI2WEYbA==",
+      "license": "MIT",
       "optional": true
     },
     "node_modules/@libsql/client": {
@@ -2505,9 +2512,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -2525,9 +2529,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -2545,9 +2546,6 @@
         "ppc64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -2565,9 +2563,6 @@
         "s390x"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -2585,9 +2580,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -2605,9 +2597,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -2714,9 +2703,6 @@
       "cpu": [
         "x64"
       ],
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -4083,6 +4069,15 @@
         "pnpm": ">=10"
       }
     },
+    "node_modules/bgutils-js": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/bgutils-js/-/bgutils-js-3.2.0.tgz",
+      "integrity": "sha512-CacO15JvxbclbLeCAAm9DETGlLuisRGWpPigoRvNsccSCPEC4pwYwA2g2x/pv7Om/sk79d4ib35V5HHmxPBpDg==",
+      "funding": [
+        "https://github.com/sponsors/LuanRT"
+      ],
+      "license": "MIT"
+    },
     "node_modules/bidi-js": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
@@ -7127,9 +7122,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -7151,9 +7143,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -7175,9 +7164,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -7199,9 +7185,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [

backend/package.json

@@ -31,6 +31,7 @@
     "@sentry/node": "^10.53.1",
     "@sentry/profiling-node": "^10.53.1",
     "better-sse": "^0.16.1",
+    "bgutils-js": "3.2.0",
     "bullmq": "^5.73.0",
     "cheerio": "^1.2.0",
     "compression": "^1.8.1",
@@ -46,6 +47,7 @@
     "helmet": "^8.2.0",
     "ioredis": "^5.10.1",
     "jiti": "^2.6.1",
+    "jsdom": "27.4.0",
     "lru-cache": "^11.3.6",
     "metascraper": "^5.50.3",
     "metascraper-author": "^5.50.1",

backend/src/services/extractors/youtube/client.ts

@@ -63,3 +63,38 @@ export async function getYoutubeClient(): Promise<Innertube> {
     throw error;
   }
 }
+
+// extraction client, armed with a poToken so ANDROID_VR returns real urls.
+// kept separate from the search client above.
+let extractorClient: Innertube | null = null;
+let extractorCreatedAt = 0;
+let extractorPoToken: string | null = null;
+
+// on by default; YT_POTOKEN=0 to disable
+const POTOKEN_ENABLED = process.env.YT_POTOKEN !== '0';
+
+export async function getYoutubeExtractorClient(): Promise<Innertube> {
+  let poToken: string | undefined;
+  let visitorData: string | undefined;
+
+  if (POTOKEN_ENABLED) {
+    const { getPoToken } = await import('./potoken.js');
+    const bundle = await getPoToken();
+    poToken = bundle?.poToken;
+    visitorData = bundle?.visitorData;
+  }
+
+  const reusable =
+    extractorClient &&
+    extractorPoToken === (poToken ?? null) &&
+    !isClientStale(extractorCreatedAt);
+  if (reusable && extractorClient) return extractorClient;
+
+  extractorClient = await _createInnertube({
+    po_token: poToken,
+    visitor_data: visitorData,
+  });
+  extractorCreatedAt = Date.now();
+  extractorPoToken = poToken ?? null;
+  return extractorClient;
+}

backend/src/services/extractors/youtube/index.ts

@@ -2,20 +2,29 @@ import { getProxiedStream } from '../../../utils/network/proxy.util.js';
 import { VideoInfo, Format, ExtractorOptions } from '../../../types/index.js';
 import { Readable } from 'node:stream';
 import { normalizeVideoInfo } from './normalizer.js';
+import { recordExtraction } from '../../../utils/infra/metrics.util.js';
 
 // skip innertube; yt-dlp handles youtube
 const YT_JS_DISABLED = process.env.DISABLE_YT_JS === '1';
 // preload client for faster first request
 const clientModule = YT_JS_DISABLED ? null : import('./client.js');
 
+// clients to try, in order. ANDROID_VR + poToken returns real urls; most
+// others are SABR-only. override with YT_JS_CLIENTS.
+type YtClient = 'ANDROID_VR' | 'IOS' | 'WEB' | 'MWEB' | 'TV' | 'ANDROID';
+const JS_CLIENTS = (process.env.YT_JS_CLIENTS || 'ANDROID_VR,IOS')
+  .split(',')
+  .map((entry) => entry.trim())
+  .filter(Boolean) as YtClient[];
+
 export async function getInfo(
   url: string,
   options: ExtractorOptions = {}
 ): Promise<VideoInfo | null> {
   if (!clientModule) return null;
   try {
-    const { getYoutubeClient } = await clientModule;
-    const client = await getYoutubeClient();
+    const { getYoutubeExtractorClient } = await clientModule;
+    const client = await getYoutubeExtractorClient();
     const idMatch = url.match(
       /(?:v=|\/v\/|youtu\.be\/|shorts\/|live\/)([0-9A-Za-z_-]{11})/u
     );
@@ -25,14 +34,53 @@ export async function getInfo(
       options.onProgress('fetching_info', 20, 'Decrypting streams...');
     }
 
-    const fullInfo = await client.getInfo(videoId);
-    const videoInfo = await normalizeVideoInfo(url, fullInfo, client);
-
-    if (options.onProgress) {
-      options.onProgress('fetching_info', 50, 'Metadata parsed');
+    /*
+     * The bug that cost me hours: youtubei.js v17 takes the client in an
+     * options object — getInfo(id, { client }). Pass a bare string like
+     * getInfo(id, 'ANDROID_VR') and it's silently ignored, so every call
+     * quietly falls back to the WEB client, which YouTube now serves
+     * SABR-only (formats but no downloadable URLs). The { client } form +
+     * a poToken is the whole difference between "JS can't get URLs" and it
+     * working fine.
+     */
+    // first client that returns real formats wins
+    for (const clientType of JS_CLIENTS) {
+      const startedAt = Date.now();
+      try {
+        const fullInfo = await client.getInfo(videoId, { client: clientType });
+        const videoInfo = await normalizeVideoInfo(url, fullInfo, client);
+        const playable =
+          (videoInfo.formats?.length || 0) +
+            (videoInfo.audioFormats?.length || 0) >
+          0;
+        recordExtraction(
+          `youtube:js:${clientType}`,
+          playable,
+          Date.now() - startedAt
+        );
+        if (playable) {
+          if (options.onProgress) {
+            options.onProgress('fetching_info', 50, 'Metadata parsed');
+          }
+          return videoInfo;
+        }
+        console.debug(
+          `[JS-YT] ${clientType} produced no playable formats for ${videoId}`
+        );
+      } catch (clientError: unknown) {
+        recordExtraction(
+          `youtube:js:${clientType}`,
+          false,
+          Date.now() - startedAt
+        );
+        console.debug(
+          `[JS-YT] ${clientType} failed: ${(clientError as Error).message}`
+        );
+      }
     }
 
-    return videoInfo;
+    // nothing usable — caller falls back to yt-dlp
+    return null;
   } catch (error: unknown) {
     const message = error instanceof Error ? error.message : String(error);
     console.error(`[JS-YT] Error extracting ${url}: ${message}`);

backend/src/services/extractors/youtube/potoken.ts

@@ -0,0 +1,106 @@
+import { Innertube } from 'youtubei.js';
+import { BG } from 'bgutils-js';
+import { JSDOM } from 'jsdom';
+
+// poToken generation. without it youtube serves SABR-only (no stream urls);
+// with it ANDROID_VR returns real urls. token is bound to visitorData and
+// lives a few hours, so generate once and cache.
+
+// well-known YouTube web BotGuard request key
+const REQUEST_KEY = 'O43z0dpjhgX20SCx4KAo';
+const DEFAULT_TTL_MS = 6 * 60 * 60 * 1000;
+const REFRESH_MARGIN_MS = 5 * 60 * 1000;
+
+export interface PoTokenBundle {
+  poToken: string;
+  visitorData: string;
+  expiresAt: number;
+}
+
+let cached: PoTokenBundle | null = null;
+let inflight: Promise<PoTokenBundle | null> | null = null;
+let domReady = false;
+
+// botguard's vm needs browser globals
+function ensureDom(): void {
+  if (domReady) return;
+  const dom = new JSDOM('<!DOCTYPE html><html><body></body></html>', {
+    url: 'https://www.youtube.com/',
+  });
+  Object.assign(globalThis, {
+    window: dom.window,
+    document: dom.window.document,
+  });
+  domReady = true;
+}
+
+async function fetchVisitorData(): Promise<string> {
+  const bootstrap = await Innertube.create({ retrieve_player: false });
+  const visitorData = bootstrap.session.context.client.visitorData;
+  if (!visitorData) throw new Error('no visitorData from bootstrap session');
+  return visitorData;
+}
+
+async function generate(): Promise<PoTokenBundle | null> {
+  try {
+    const visitorData = await fetchVisitorData();
+    ensureDom();
+
+    const bgConfig = {
+      fetch: (...args: Parameters<typeof fetch>) => fetch(...args),
+      globalObj: globalThis as unknown as Record<string, unknown>,
+      identifier: visitorData,
+      requestKey: REQUEST_KEY,
+    } as Parameters<typeof BG.Challenge.create>[0];
+
+    const challenge = await BG.Challenge.create(bgConfig);
+    if (!challenge) throw new Error('challenge creation returned nothing');
+
+    const script =
+      challenge.interpreterJavascript
+        .privateDoNotAccessOrElseSafeScriptWrappedValue;
+    if (!script) throw new Error('challenge missing interpreter script');
+
+    // run the botguard vm (bgutils needs it in the global realm)
+    // eslint-disable-next-line sonarjs/code-eval -- trusted botguard payload
+    new Function(script)();
+
+    const result = await BG.PoToken.generate({
+      program: challenge.program,
+      globalName: challenge.globalName,
+      bgConfig,
+    });
+
+    const ttlSecs = result.integrityTokenData?.estimatedTtlSecs;
+    const ttlMs = ttlSecs ? ttlSecs * 1000 : DEFAULT_TTL_MS;
+    const bundle: PoTokenBundle = {
+      poToken: result.poToken,
+      visitorData,
+      expiresAt: Date.now() + Math.max(ttlMs - REFRESH_MARGIN_MS, 60_000),
+    };
+    cached = bundle;
+    console.log(
+      `[poToken] generated (len=${bundle.poToken.length}, ttl=${Math.round(
+        (bundle.expiresAt - Date.now()) / 1000
+      )}s)`
+    );
+    return bundle;
+  } catch (err) {
+    console.warn('[poToken] generation failed:', (err as Error).message);
+    return null;
+  }
+}
+
+// cached token; regenerates on expiry, one in-flight gen at a time, null on failure
+export function getPoToken(
+  forceRefresh = false
+): Promise<PoTokenBundle | null> {
+  if (!forceRefresh && cached && Date.now() < cached.expiresAt) {
+    return Promise.resolve(cached);
+  }
+  if (inflight) return inflight;
+  inflight = generate().finally(() => {
+    inflight = null;
+  });
+  return inflight;
+}

backend/src/services/ytdlp/info-core.ts

@@ -13,6 +13,7 @@ import {
   processAudioFormats,
 } from '../../utils/media/format.util.js';
 import { createRedisClient } from '../../utils/infra/redis.util.js';
+import { recordExtraction } from '../../utils/infra/metrics.util.js';
 import { LRUCache } from 'lru-cache';
 
 export type ProgressCallback = (
@@ -322,6 +323,7 @@ export async function runYtdlpInfo(
   signal: AbortSignal | null = null,
   isRetry = false
 ): Promise<VideoInfo> {
+  const ytdlpStart = Date.now();
   const refererResult = Object.entries(REFERER_MAP).find(([domain]) =>
     targetUrl.includes(domain)
   );
@@ -403,5 +405,12 @@ export async function runYtdlpInfo(
   // cache for subsequent stream spawn
   void persistInfoJsonToDisk(parsedData, stdout);
 
+  const ytdlpLabel = targetUrl.includes('tiktok.com')
+    ? 'tiktok:ytdlp'
+    : targetUrl.includes('youtube.com') || targetUrl.includes('youtu.be')
+      ? 'youtube:ytdlp'
+      : 'other:ytdlp';
+  recordExtraction(ytdlpLabel, true, Date.now() - ytdlpStart);
+
   return parsedData;
 }

backend/src/types/custom.d.ts

@@ -8,3 +8,4 @@ declare module 'jiti';
 declare module 'sucrase';
 declare module 'node:vm';
 declare module 'better-sse';
+declare module 'jsdom';

backend/src/utils/infra/metrics.util.ts

@@ -17,6 +17,10 @@ export interface MetricsSnapshot {
     { count: number; errors: number; avgMs: number; maxMs: number }
   >;
   failures: Record<string, number>;
+  extractions: Record<
+    string,
+    { attempts: number; success: number; successRate: number; avgMs: number; maxMs: number }
+  >;
 }
 
 // bound label cardinality against abuse
@@ -25,6 +29,14 @@ const startedAt = Date.now();
 const routes = new Map<string, RouteStat>();
 const failures = new Map<string, number>();
 
+interface ExtractionStat {
+  attempts: number;
+  success: number;
+  totalMs: number;
+  maxMs: number;
+}
+const extractions = new Map<string, ExtractionStat>();
+
 // fold overflow keys into "other"
 function capKey(store: Map<string, unknown>, rawKey: string): string {
   if (store.has(rawKey) || store.size < MAX_LABELS) {
@@ -61,6 +73,30 @@ export function recordFailure(reason: string): void {
   failures.set(key, (failures.get(key) ?? 0) + 1);
 }
 
+// track extraction outcomes (youtube:js:ANDROID_VR vs youtube:ytdlp) for /metrics
+export function recordExtraction(
+  label: string,
+  success: boolean,
+  durationMs: number
+): void {
+  const key = capKey(extractions, label);
+  const stat = extractions.get(key) ?? {
+    attempts: 0,
+    success: 0,
+    totalMs: 0,
+    maxMs: 0,
+  };
+  stat.attempts += 1;
+  if (success) {
+    stat.success += 1;
+  }
+  stat.totalMs += durationMs;
+  if (durationMs > stat.maxMs) {
+    stat.maxMs = durationMs;
+  }
+  extractions.set(key, stat);
+}
+
 export function getMetrics(): MetricsSnapshot {
   const perRoute: MetricsSnapshot['routes'] = {};
   let totalRequests = 0;
@@ -81,12 +117,27 @@ export function getMetrics(): MetricsSnapshot {
     totalErrors,
     routes: perRoute,
     failures: Object.fromEntries(failures),
+    extractions: Object.fromEntries(
+      [...extractions].map(([label, stat]) => [
+        label,
+        {
+          attempts: stat.attempts,
+          success: stat.success,
+          successRate: stat.attempts
+            ? Math.round((stat.success / stat.attempts) * 100) / 100
+            : 0,
+          avgMs: Math.round(stat.totalMs / stat.attempts),
+          maxMs: stat.maxMs,
+        },
+      ])
+    ),
   };
 }
 
 export function resetMetrics(): void {
   routes.clear();
   failures.clear();
+  extractions.clear();
 }
 
 // record latency + outcome per request