deploy 41a98f4

backend/package.json

@@ -13,9 +13,9 @@
     "test:lite": "npx tsc && node --test tests/lite/*.lite.test.js",
     "test:ci": "./scripts/ci-test.sh",
     "test:ci:keep": "./scripts/ci-test.sh --keep",
-    "test:single": "NODE_OPTIONS='--import ./scripts/termux-shim.js --max-old-space-size=512' npx vitest run",
+    "test:single": "NODE_ENV=test NODE_OPTIONS='--import ./scripts/termux-shim.js --max-old-space-size=512' npx vitest run",
     "verify:chunked": "VITEST_INCLUDE_MANUAL=1 SMOKE_TEST=1 npx vitest run tests/manual/chunked_fetcher_smoke.test.ts --no-coverage",
-    "test:live": "VITEST_INCLUDE_MANUAL=1 LIVE_TEST=1 npx vitest run tests/manual/facebook_live.test.ts tests/manual/instagram_live.test.ts --no-coverage",
+    "test:live": "VITEST_INCLUDE_MANUAL=1 LIVE_TEST=1 npx vitest run tests/manual/facebook_live.test.ts tests/manual/instagram_live.test.ts tests/manual/tiktok_live.test.ts tests/manual/x_live.test.ts tests/manual/soundcloud_live.test.ts --no-coverage",
     "bench:convert": "node scripts/bench-convert.js",
     "lint": "bash ../scripts/lint-changed.sh",
     "lint:all": "eslint .",

backend/scripts/termux-shim.js

@@ -31,5 +31,7 @@ Module.prototype.require = function (name, ...args) {
 
 console.log('[env] termux bypass active');
 
-// load main app
-await import('../dist/backend/src/app.js');
+// skip server boot under tests
+if (process.env.NODE_ENV !== 'test' && !process.env.VITEST) {
+  await import('../dist/backend/src/app.js');
+}

backend/scripts/test-sequential.sh

@@ -30,7 +30,7 @@ for i in "${!FILES[@]}"; do
   IDX=$((i + 1))
   printf "[%d/%d] %s ... " "$IDX" "$TOTAL" "$FILE"
 
-  if NODE_OPTIONS='--import ./scripts/termux-shim.js --max-old-space-size=384' \
+  if NODE_ENV=test NODE_OPTIONS='--import ./scripts/termux-shim.js --max-old-space-size=384' \
      npx vitest run "$FILE" --reporter=dot --no-coverage > /tmp/vitest-out.log 2>&1; then
     echo "PASS"
     PASS=$((PASS + 1))

backend/src/services/extractors/instagram/constants.ts

@@ -0,0 +1,37 @@
+// required by instagram private api
+export const IG_APP_ID = '936619743392459';
+
+// graphql doc id for post query
+export const POST_DOC_ID = '8845758582119845';
+
+export const DESKTOP_UA =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+
+// android app ua for mobile api
+export const MOBILE_UA =
+  'Instagram 275.0.0.27.98 Android (33/13; 280dpi; 720x1423; Xiaomi; Redmi 7; onclite; qcom; en_US; 458229237)';
+
+// app-id unlocks logged-out web json
+export const WEB_HEADERS: Record<string, string> = {
+  'User-Agent': DESKTOP_UA,
+  'x-ig-app-id': IG_APP_ID,
+  'Accept-Language': 'en-US,en;q=0.9',
+  'Sec-Fetch-Site': 'same-origin',
+};
+
+export const MOBILE_HEADERS: Record<string, string> = {
+  'User-Agent': MOBILE_UA,
+  'x-ig-app-id': IG_APP_ID,
+  'x-ig-app-locale': 'en_US',
+  'x-ig-device-locale': 'en_US',
+  'x-ig-mapped-locale': 'en_US',
+  'Accept-Language': 'en-US',
+  'x-fb-http-engine': 'Liger',
+};
+
+export const EMBED_HEADERS: Record<string, string> = {
+  'User-Agent': DESKTOP_UA,
+  Accept:
+    'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+  'Accept-Language': 'en-GB,en;q=0.9',
+};

backend/src/services/extractors/instagram/fetcher.ts

@@ -1,61 +1,117 @@
 import { secureFetch } from '../../../utils/network/security.util.js';
+import { ExtractorOptions } from '../../../types/index.js';
+import {
+  WEB_HEADERS,
+  MOBILE_HEADERS,
+  EMBED_HEADERS,
+  POST_DOC_ID,
+} from './constants.js';
 
-export async function fetchJson(
-  url: string,
-  options: { cookie?: string } = {}
-): Promise<unknown> {
+const TIMEOUT_MS = 10000;
+
+// pull shortcode from any post/reel/tv url
+export function extractShortcode(url: string): string | null {
+  const match = url.match(/\/(?:reels?|p|tv)\/([A-Za-z0-9_-]+)/u);
+  return match ? match[1] : null;
+}
+
+function withCookie(
+  headers: Record<string, string>,
+  options: ExtractorOptions
+): Record<string, string> {
   const cookie = typeof options.cookie === 'string' ? options.cookie : null;
-  const response = await secureFetch(url, {
-    headers: {
-      'User-Agent':
-        'Mozilla/5.0 (iPhone; CPU iPhone OS 14_8 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1',
-      Accept: '*/*',
-      'Accept-Language': 'en-US,en;q=0.5',
-      'X-Requested-With': 'XMLHttpRequest',
-      ...(cookie && { Cookie: cookie }),
-    },
-    signal: AbortSignal.timeout(10000),
+  return cookie ? { ...headers, Cookie: cookie } : headers;
+}
+
+// oembed maps shortcode to numeric media id
+async function fetchMediaId(
+  shortcode: string,
+  options: ExtractorOptions
+): Promise<string | null> {
+  const oembedUrl = `https://i.instagram.com/api/v1/oembed/?url=https://www.instagram.com/p/${shortcode}/`;
+  const response = await secureFetch(oembedUrl, {
+    headers: withCookie(MOBILE_HEADERS, options),
+    signal: AbortSignal.timeout(TIMEOUT_MS),
   });
+  if (!response.ok) return null;
+  const data = (await response.json()) as { media_id?: string };
+  return data?.media_id ?? null;
+}
+
+// primary path, mobile private api
+export async function fetchMobileItem(
+  shortcode: string,
+  options: ExtractorOptions
+): Promise<unknown> {
+  const mediaId = await fetchMediaId(shortcode, options);
+  if (!mediaId) return null;
 
+  const response = await secureFetch(
+    `https://i.instagram.com/api/v1/media/${mediaId}/info/`,
+    {
+      headers: withCookie(MOBILE_HEADERS, options),
+      signal: AbortSignal.timeout(TIMEOUT_MS),
+    }
+  );
   if (!response.ok) return null;
-  return await response.json();
+  const data = (await response.json()) as { items?: unknown[] };
+  return data?.items?.[0] ?? null;
 }
 
-export async function fetchEmbed(
-  url: string,
-  options: { cookie?: string } = {}
-): Promise<string | null> {
-  const cookie = typeof options.cookie === 'string' ? options.cookie : null;
-  const response = await secureFetch(`${url}embed/captioned/`, {
+// secondary path, web graphql
+export async function fetchGraphqlMedia(
+  shortcode: string,
+  options: ExtractorOptions
+): Promise<unknown> {
+  const body = new URLSearchParams({
+    doc_id: POST_DOC_ID,
+    variables: JSON.stringify({ shortcode }),
+  });
+
+  const response = await secureFetch('https://www.instagram.com/graphql/query', {
+    method: 'POST',
     headers: {
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-      ...(cookie && { Cookie: cookie }),
+      ...withCookie(WEB_HEADERS, options),
+      'Content-Type': 'application/x-www-form-urlencoded',
+      'X-FB-Friendly-Name': 'PolarisPostActionLoadPostQueryQuery',
     },
-    signal: AbortSignal.timeout(10000),
+    body: body.toString(),
+    signal: AbortSignal.timeout(TIMEOUT_MS),
   });
+  if (!response.ok) return null;
+  const json = (await response.json()) as {
+    data?: { xdt_shortcode_media?: unknown; shortcode_media?: unknown };
+  };
+  return json?.data?.xdt_shortcode_media ?? json?.data?.shortcode_media ?? null;
+}
 
+// last resort, captioned embed html
+export async function fetchEmbedHtml(
+  url: string,
+  options: ExtractorOptions
+): Promise<string | null> {
+  const base = url.split('?')[0].replace(/\/?$/u, '/');
+  const response = await secureFetch(`${base}embed/captioned/`, {
+    headers: withCookie(EMBED_HEADERS, options),
+    signal: AbortSignal.timeout(TIMEOUT_MS),
+  });
   if (!response.ok) return null;
   return await response.text();
 }
 
 export async function fetchFileSize(url: string): Promise<number | undefined> {
   try {
-    const headResponse = await secureFetch(url, {
+    const response = await secureFetch(url, {
       method: 'HEAD',
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-      },
+      headers: { 'User-Agent': WEB_HEADERS['User-Agent'] },
       signal: AbortSignal.timeout(5000),
     });
-    if (headResponse.ok) {
-      const contentLengthHeader = headResponse.headers.get('content-length');
-      if (contentLengthHeader) return parseInt(contentLengthHeader, 10);
+    if (response.ok) {
+      const len = response.headers.get('content-length');
+      if (len) return parseInt(len, 10);
     }
   } catch (error: unknown) {
-    const errorObj = error as Error;
-    console.debug(`[Instagram] Size fetch failed: ${errorObj.message}`);
+    console.debug(`[Instagram] Size fetch failed: ${(error as Error).message}`);
   }
   return undefined;
 }

backend/src/services/extractors/instagram/index.ts

@@ -1,45 +1,69 @@
 import { getQuantumStream } from '../../../utils/network/proxy.util.js';
 import { VideoInfo, Format, ExtractorOptions } from '../../../types/index.js';
+import { IgParsed } from './types.js';
 import { Readable } from 'node:stream';
-import { fetchJson, fetchEmbed, fetchFileSize } from './fetcher.js';
-import { parseJson, parseEmbed } from './parser.js';
+import { DESKTOP_UA } from './constants.js';
+import {
+  extractShortcode,
+  fetchMobileItem,
+  fetchGraphqlMedia,
+  fetchEmbedHtml,
+  fetchFileSize,
+} from './fetcher.js';
+import { parseMobileItem, parseGraphqlMedia, parseEmbed } from './parser.js';
 import { normalizeVideoInfo } from './normalizer.js';
 
+async function resolveParsed(
+  url: string,
+  options: ExtractorOptions
+): Promise<IgParsed | null> {
+  const shortcode = extractShortcode(url);
+
+  // ordered cascade, first with media wins
+  const resolvers: Array<() => Promise<IgParsed | null>> = [];
+  if (shortcode) {
+    resolvers.push(async () =>
+      parseMobileItem(await fetchMobileItem(shortcode, options))
+    );
+    resolvers.push(async () =>
+      parseGraphqlMedia(await fetchGraphqlMedia(shortcode, options))
+    );
+  }
+  resolvers.push(async () => {
+    const html = await fetchEmbedHtml(url, options);
+    return html ? parseEmbed(html) : null;
+  });
+
+  for (const resolve of resolvers) {
+    try {
+      const parsed = await resolve();
+      if (parsed && parsed.media.length > 0) return parsed;
+    } catch (error: unknown) {
+      console.debug(`[JS-IG] path failed: ${(error as Error).message}`);
+    }
+  }
+  return null;
+}
+
 export async function getInfo(
   url: string,
   options: ExtractorOptions = {}
 ): Promise<VideoInfo | null> {
   try {
-    // try JSON
-    let rawData = null;
-    const jsonUrl = url.includes('?')
-      ? `${url.split('?')[0]}?__a=1&__d=dis`
-      : `${url}?__a=1&__d=dis`;
-    const jsonData = await fetchJson(jsonUrl, options);
-    if (jsonData) {
-      rawData = parseJson(jsonData);
-    }
-
-    if (!rawData) {
-      const embedHtml = await fetchEmbed(url, options);
-      if (embedHtml) {
-        rawData = parseEmbed(embedHtml);
-      }
-    }
-
-    if (!rawData) return null;
+    const parsed = await resolveParsed(url, options);
+    if (!parsed) return null;
 
-    const videoInfo = normalizeVideoInfo(url, rawData);
+    const videoInfo = normalizeVideoInfo(url, parsed);
     if (!videoInfo) return null;
 
-    // fetch size
-    for (let index = 0; index < videoInfo.formats.length; index += 2) {
-      const batch = videoInfo.formats.slice(index, index + 2);
+    // sizes are optional, fetch in small batches
+    for (let index = 0; index < videoInfo.formats.length; index += 3) {
+      const batch = videoInfo.formats.slice(index, index + 3);
       await Promise.all(
-        batch.map(async (formatItem: Format) => {
-          if (formatItem.url) {
-            const size = await fetchFileSize(formatItem.url);
-            if (size) formatItem.filesize = size;
+        batch.map(async (format: Format) => {
+          if (format.url) {
+            const size = await fetchFileSize(format.url);
+            if (size) format.filesize = size;
           }
         })
       );
@@ -66,10 +90,10 @@ export function getStream(
 
   return Promise.resolve(
     getQuantumStream(targetFormat.url, {
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'User-Agent': DESKTOP_UA,
       Referer: 'https://www.instagram.com/',
       Accept: '*/*',
+      'Accept-Language': 'en-US,en;q=0.9',
     })
   );
 }

backend/src/services/extractors/instagram/normalizer.ts

@@ -1,32 +1,56 @@
-import { VideoInfo } from '../../../types/index.js';
+import { VideoInfo, Format } from '../../../types/index.js';
 import { normalizeTitle, normalizeArtist } from '../../social.service.js';
+import { IgParsed, IgMedia } from './types.js';
 
-/* eslint-disable @typescript-eslint/no-explicit-any */
-export function normalizeVideoInfo(
-  url: string,
-  parsedData: any
-): VideoInfo | null {
-  if (!parsedData) return null;
+function toFormat(media: IgMedia, index: number, total: number): Format {
+  const dims =
+    media.width && media.height ? `${media.width}x${media.height}` : undefined;
+  // label carousel children for the picker
+  const prefix = total > 1 ? `item${index + 1}_` : '';
 
-  const rawFormats = (parsedData.formats || []) as any[];
-  const formats = rawFormats.map((formatItem) => {
+  if (media.isVideo) {
     return {
-      formatId: formatItem.id || `ig_${Math.random().toString(36).slice(2, 7)}`,
-      url: formatItem.video_url || formatItem.display_url,
+      formatId: `${prefix}hd`,
+      url: media.url,
       extension: 'mp4',
-      resolution: formatItem.width
-        ? `${formatItem.width}x${formatItem.height}`
-        : 'Source',
-      width: formatItem.width,
-      height: formatItem.height,
-      filesize: formatItem.filesize,
-      acodec: 'aac',
+      resolution: dims ?? 'Source',
+      quality: total > 1 ? `Item ${index + 1}` : 'HD',
+      width: media.width,
+      height: media.height,
       vcodec: 'h264',
-      isAudio: true,
-      isVideo: true,
+      acodec: 'aac',
       isMuxed: true,
+      isVideo: true,
+      isAudio: true,
     };
-  });
+  }
+
+  return {
+    formatId: `${prefix}photo`,
+    url: media.url,
+    extension: 'jpg',
+    resolution: dims ?? 'Photo',
+    quality: total > 1 ? `Item ${index + 1}` : 'Photo',
+    width: media.width,
+    height: media.height,
+    vcodec: 'none',
+    isMuxed: false,
+    isVideo: false,
+    isAudio: false,
+  };
+}
+
+export function normalizeVideoInfo(
+  url: string,
+  parsedData: IgParsed | null
+): VideoInfo | null {
+  if (!parsedData) return null;
+
+  const total = parsedData.media.length;
+  const formats: Format[] = parsedData.media.map((media, index) =>
+    toFormat(media, index, total)
+  );
+  if (formats.length === 0) return null;
 
   const info: VideoInfo = {
     type: 'video',
@@ -34,15 +58,21 @@ export function normalizeVideoInfo(
     title: parsedData.title || 'Instagram Video',
     uploader: parsedData.uploader || 'Instagram User',
     webpageUrl: url,
+    thumbnail: parsedData.thumbnail,
     formats,
     extractorKey: 'instagram',
     isJsInfo: true,
     fromBrain: false,
     isPartial: false,
     isIsrcMatch: false,
-    isFullData: false,
+    isFullData: true,
   };
 
+  // make caption authoritative over og:title
+  if (parsedData.title) {
+    info.metascraper = { title: parsedData.title };
+  }
+
   info.title = normalizeTitle(info as unknown as Record<string, unknown>);
   info.uploader = normalizeArtist(info as unknown as Record<string, unknown>);
 

backend/src/services/extractors/instagram/parser.ts

@@ -1,70 +1,137 @@
-import { load, CheerioAPI } from 'cheerio';
+import { load } from 'cheerio';
 import { decode } from '../facebook/utils.js';
+import { IgParsed, IgMedia } from './types.js';
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
-export function parseJson(jsonData: any): unknown {
-  const items = (jsonData?.items || []) as any[];
-  if (items.length === 0) return null;
 
-  const item = items[0];
-  const videoVersions = (item.video_versions || []) as any[];
-  const user = item.user;
+// highest pixel-count rendition wins
+function bestVideo(versions: any[]): any {
+  return versions.reduce((prev, next) =>
+    (prev.width ?? 0) * (prev.height ?? 0) <
+    (next.width ?? 0) * (next.height ?? 0)
+      ? next
+      : prev
+  );
+}
+
+function mediaFromMobile(node: any): IgMedia | null {
+  const videos = node?.video_versions;
+  if (Array.isArray(videos) && videos.length > 0) {
+    const best = bestVideo(videos);
+    return {
+      url: best.url,
+      isVideo: true,
+      width: best.width,
+      height: best.height,
+    };
+  }
+  const candidate = node?.image_versions2?.candidates?.[0];
+  if (candidate?.url) {
+    return {
+      url: candidate.url,
+      isVideo: false,
+      width: candidate.width,
+      height: candidate.height,
+    };
+  }
+  return null;
+}
+
+function mediaFromGql(node: any): IgMedia | null {
+  if (node?.video_url) {
+    return {
+      url: node.video_url,
+      isVideo: true,
+      width: node.dimensions?.width,
+      height: node.dimensions?.height,
+    };
+  }
+  if (node?.display_url) {
+    return {
+      url: node.display_url,
+      isVideo: false,
+      width: node.dimensions?.width,
+      height: node.dimensions?.height,
+    };
+  }
+  return null;
+}
+
+// mobile api items[0], single or carousel
+export function parseMobileItem(item: any): IgParsed | null {
+  if (!item) return null;
+
+  const carousel = item.carousel_media;
+  const media: IgMedia[] = Array.isArray(carousel)
+    ? (carousel.map(mediaFromMobile).filter(Boolean) as IgMedia[])
+    : ([mediaFromMobile(item)].filter(Boolean) as IgMedia[]);
+
+  if (media.length === 0) return null;
 
   return {
-    id: item.pk || item.id,
+    id: item.code || item.pk || item.id || null,
     title: item.caption?.text || 'Instagram Post',
-    uploader: user?.full_name || user?.username,
+    uploader: item.user?.full_name || item.user?.username || 'Instagram User',
     thumbnail: item.image_versions2?.candidates?.[0]?.url,
-    formats: videoVersions.map((version) => ({
-      id: version.id,
-      video_url: version.url,
-      width: version.width,
-      height: version.height,
-    })),
+    media,
   };
 }
 
-const _extractThumbnailFromCheerio = (cheerioInstance: CheerioAPI) => {
-  return (
-    cheerioInstance('meta[property="og:image"]').attr('content') ||
-    cheerioInstance('img.EmbeddedMediaImage').attr('src')
-  );
-};
+// graphql shortcode_media, single or sidecar
+export function parseGraphqlMedia(node: any): IgParsed | null {
+  if (!node) return null;
+
+  const sidecar = node.edge_sidecar_to_children?.edges;
+  const media: IgMedia[] = Array.isArray(sidecar)
+    ? (sidecar.map((edge: any) => mediaFromGql(edge?.node)).filter(Boolean) as IgMedia[])
+    : ([mediaFromGql(node)].filter(Boolean) as IgMedia[]);
+
+  if (media.length === 0) return null;
+
+  return {
+    id: node.shortcode || node.id || null,
+    title: node.edge_media_to_caption?.edges?.[0]?.node?.text || 'Instagram Post',
+    uploader: node.owner?.full_name || node.owner?.username || 'Instagram User',
+    thumbnail: node.display_url,
+    media,
+  };
+}
+
+// embed bundles a graphql node in contextJSON
+function extractEmbedContext(html: string): any {
+  try {
+    const initMatch = html.match(/"init",\[\],\[(.*?)\]\],/u);
+    if (!initMatch) return null;
+    const init = JSON.parse(initMatch[1]);
+    if (!init?.contextJSON) return null;
+    const ctx = JSON.parse(init.contextJSON);
+    return ctx?.gql_data?.shortcode_media || ctx?.gql_data?.xdt_shortcode_media || null;
+  } catch {
+    return null;
+  }
+}
+
+export function parseEmbed(html: string): IgParsed | null {
+  const ctx = extractEmbedContext(html);
+  if (ctx) {
+    const structured = parseGraphqlMedia(ctx);
+    if (structured) return structured;
+  }
 
-const _extractVideoUrlFromHtml = (html: string) => {
+  // degraded regex path for older embeds
   const videoMatch = html.match(/"video_url"\s*:\s*"([^"]+)"/u);
-  return videoMatch ? decode(videoMatch[1]) : null;
-};
-
-export function parseEmbed(html: string): unknown {
-  const cheerioInstance = load(html);
-  const title =
-    cheerioInstance('meta[property="og:title"]').attr('content') ||
-    cheerioInstance('.Caption').text() ||
-    'Instagram Video';
-
-  const uploader =
-    cheerioInstance('.Username').text() ||
-    cheerioInstance('.UsernameText').text() ||
-    'Instagram User';
-
-  const thumbnail = _extractThumbnailFromCheerio(cheerioInstance);
-  const videoUrl = _extractVideoUrlFromHtml(html);
-
-  const formats = videoUrl
-    ? [
-        {
-          id: 'hd',
-          video_url: videoUrl,
-        },
-      ]
-    : [];
+  const url = videoMatch ? decode(videoMatch[1]) : null;
+  if (!url) return null;
 
+  const page = load(html);
   return {
     id: null,
-    title,
-    uploader,
-    thumbnail,
-    formats,
+    title:
+      page('meta[property="og:title"]').attr('content') ||
+      page('.Caption').text() ||
+      'Instagram Video',
+    uploader: page('.Username').text() || 'Instagram User',
+    thumbnail: page('meta[property="og:image"]').attr('content'),
+    media: [{ url, isVideo: true }],
   };
 }

backend/src/services/extractors/instagram/types.ts

@@ -0,0 +1,16 @@
+// one resolved media (carousel child or single)
+export interface IgMedia {
+  url: string;
+  isVideo: boolean;
+  width?: number;
+  height?: number;
+}
+
+// normalized shape across all fetch paths
+export interface IgParsed {
+  id: string | null;
+  title: string;
+  uploader: string;
+  thumbnail?: string;
+  media: IgMedia[];
+}

backend/tests/extractors/instagram_extractor.test.ts

@@ -3,6 +3,9 @@ import {
   getInfo,
   getStream,
 } from '../../src/services/extractors/instagram/index.js';
+import { IG_APP_ID } from '../../src/services/extractors/instagram/constants.js';
+import { server } from '../setup.js';
+import { http, HttpResponse } from 'msw';
 import { Readable } from 'node:stream';
 import { z } from 'zod';
 import { CaseSchema } from '../utils/schema.js';
@@ -10,6 +13,8 @@ import { assertOutcome } from '../utils/assert.js';
 import rawCases from '../fixtures/extractors/instagram.json';
 
 const testCases = z.array(CaseSchema).parse(rawCases);
+const REEL = 'https://www.instagram.com/reel/DFQe23tOWKz/';
+const MEDIA_INFO = 'https://i.instagram.com/api/v1/media/:mediaId/info/';
 
 describe('Instagram JS Extractor (Data-Driven)', () => {
   beforeEach(() => {
@@ -19,10 +24,13 @@ describe('Instagram JS Extractor (Data-Driven)', () => {
   it.each(testCases)('should extract metadata for $name', async (testCase) => {
     const info = await getInfo(testCase.url);
     assertOutcome(info, testCase.expected);
-    
-    if (testCase.expected.status === 'ok' && testCase.expected.type === 'video') {
-       expect(info?.formats?.length).toBeGreaterThan(0);
-       expect(info?.formats[0].url).toContain('test.mp4');
+
+    if (
+      testCase.expected.status === 'ok' &&
+      testCase.expected.type === 'video'
+    ) {
+      expect(info?.formats?.length).toBeGreaterThan(0);
+      expect(info?.formats[0].url).toContain('test.mp4');
     }
   });
 
@@ -39,3 +47,116 @@ describe('Instagram JS Extractor (Data-Driven)', () => {
     stream.destroy();
   });
 });
+
+describe('Instagram Gold Standard (2026)', () => {
+  it('sends the mandatory X-IG-App-ID header', async () => {
+    let sentAppId: string | null = null;
+    server.use(
+      http.get('https://i.instagram.com/api/v1/oembed/', ({ request }) => {
+        sentAppId = request.headers.get('x-ig-app-id');
+        return HttpResponse.json({ media_id: '123456_789' });
+      })
+    );
+
+    await getInfo(REEL);
+    expect(sentAppId).toBe(IG_APP_ID);
+    expect(sentAppId).toBe('936619743392459');
+  });
+
+  it('selects the highest-quality video version', async () => {
+    const info = await getInfo(REEL);
+    // mock serves sd 480x854 + hd 1080x1920
+    expect(info?.formats[0].resolution).toBe('1080x1920');
+    expect(info?.formats[0].url).toContain('test.mp4');
+    expect(info?.formats[0].url).not.toContain('test_sd');
+  });
+
+  it('exposes each carousel child as a format', async () => {
+    server.use(
+      http.get(MEDIA_INFO, () =>
+        HttpResponse.json({
+          items: [
+            {
+              code: 'CAROUSEL1',
+              caption: { text: 'Carousel Post' },
+              user: { username: 'test_user' },
+              carousel_media: [
+                {
+                  video_versions: [
+                    {
+                      url: 'https://scontent.cdninstagram.com/v/c1.mp4',
+                      width: 1080,
+                      height: 1080,
+                    },
+                  ],
+                },
+                {
+                  image_versions2: {
+                    candidates: [
+                      {
+                        url: 'https://scontent.cdninstagram.com/c2.jpg',
+                        width: 1080,
+                        height: 1080,
+                      },
+                    ],
+                  },
+                },
+              ],
+            },
+          ],
+        })
+      )
+    );
+
+    const info = await getInfo(REEL);
+    expect(info?.formats).toHaveLength(2);
+    expect(info?.formats[0].formatId).toBe('item1_hd');
+    expect(info?.formats[1].formatId).toBe('item2_photo');
+    expect(info?.formats[1].extension).toBe('jpg');
+  });
+
+  it('handles image-only posts as photo formats', async () => {
+    server.use(
+      http.get(MEDIA_INFO, () =>
+        HttpResponse.json({
+          items: [
+            {
+              code: 'PHOTO1',
+              caption: { text: 'Photo Post' },
+              user: { username: 'test_user' },
+              image_versions2: {
+                candidates: [
+                  {
+                    url: 'https://scontent.cdninstagram.com/photo.jpg',
+                    width: 1080,
+                    height: 1350,
+                  },
+                ],
+              },
+            },
+          ],
+        })
+      )
+    );
+
+    const info = await getInfo(REEL);
+    expect(info?.formats[0].extension).toBe('jpg');
+    expect(info?.formats[0].vcodec).toBe('none');
+    expect(info?.formats[0].isVideo).toBe(false);
+  });
+
+  it('falls back to graphql when the mobile api fails', async () => {
+    server.use(
+      http.get(
+        'https://i.instagram.com/api/v1/oembed/',
+        () => new HttpResponse(null, { status: 404 })
+      ),
+      http.get(MEDIA_INFO, () => new HttpResponse(null, { status: 404 }))
+    );
+
+    const info = await getInfo(REEL);
+    expect(info).not.toBeNull();
+    expect(info?.formats?.length).toBeGreaterThan(0);
+    expect(info?.formats[0].url).toContain('test.mp4');
+  });
+});

backend/tests/fixtures/live-extractor-urls.json

@@ -8,12 +8,12 @@
     "note": "public reel; IG may need cookies even on a residential IP"
   },
   "x": {
-    "url": "https://x.com/farmskins/status/2044702637853905192",
+    "url": "https://x.com/earthshakerph/status/2063795469898444995",
     "note": "public tweet with video"
   },
   "tiktok": {
-    "url": "https://www.tiktok.com/@scout2015/video/6718335390845095173",
-    "note": "tiktok oEmbed example video; long-lived"
+    "url": "https://www.tiktok.com/@gablyricsssss/video/7636385053227879700",
+    "note": "public tiktok video"
   },
   "soundcloud": {
     "url": "https://soundcloud.com/forss/flickermood",

backend/tests/manual/soundcloud_live.test.ts

@@ -0,0 +1,42 @@
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'node:fs';
+import { soundcloud } from '../../src/services/extractors/index.js';
+
+// live test, real soundcloud fetch
+// run on residential ip not datacenter
+const RUN = process.env.LIVE_TEST === '1';
+const ldescribe = RUN ? describe : describe.skip;
+
+interface LiveUrlEntry {
+  url: string;
+  note?: string;
+}
+
+// volatile urls live in one editable json
+const liveUrls = JSON.parse(
+  readFileSync(
+    new URL('../fixtures/live-extractor-urls.json', import.meta.url),
+    'utf8'
+  )
+) as Record<string, LiveUrlEntry>;
+
+const SC_URL = process.env.SOUNDCLOUD_LIVE_URL || liveUrls.soundcloud?.url;
+
+ldescribe('soundcloud extractor (live)', () => {
+  it('resolves a real soundcloud track to playable formats', async () => {
+    expect(SC_URL, 'no soundcloud url in fixtures').toBeTruthy();
+    const info = await soundcloud.getInfo(SC_URL, {});
+
+    expect(info, 'extractor returned null — likely broken').toBeTruthy();
+    expect(info?.title, 'no title resolved').toBeTruthy();
+    // canary for soundcloud api changes
+    expect(
+      info?.formats?.length ?? 0,
+      'no formats — soundcloud likely changed, extractor needs a fix'
+    ).toBeGreaterThan(0);
+
+    console.log(
+      `[live] soundcloud OK: "${info?.title}" — ${info?.formats?.length} format(s)`
+    );
+  }, 60000);
+});

backend/tests/manual/tiktok_live.test.ts

@@ -0,0 +1,78 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { readFileSync } from 'node:fs';
+import {
+  getGlobalDispatcher,
+  setGlobalDispatcher,
+  ProxyAgent,
+} from 'undici';
+import type { Dispatcher } from 'undici';
+import { tiktok } from '../../src/services/extractors/index.js';
+
+// live test, real tiktok fetch
+// run on residential ip not datacenter
+const RUN = process.env.LIVE_TEST === '1';
+const ldescribe = RUN ? describe : describe.skip;
+
+// tiktok captchas direct ips; proxy unblocks
+const PROXY = process.env.LIVE_PROXY;
+const TT_ENABLED = Boolean(PROXY) || process.env.TIKTOK_LIVE === '1';
+
+interface LiveUrlEntry {
+  url: string;
+  note?: string;
+}
+
+// volatile urls live in one editable json
+const liveUrls = JSON.parse(
+  readFileSync(
+    new URL('../fixtures/live-extractor-urls.json', import.meta.url),
+    'utf8'
+  )
+) as Record<string, LiveUrlEntry>;
+
+const TT_URL = process.env.TIKTOK_LIVE_URL || liveUrls.tiktok?.url;
+
+ldescribe('tiktok extractor (live)', () => {
+  let original: Dispatcher | undefined;
+
+  // route extractor fetches through residential proxy
+  beforeAll(() => {
+    if (PROXY) {
+      original = getGlobalDispatcher();
+      setGlobalDispatcher(new ProxyAgent(PROXY));
+    }
+  });
+
+  // restore so proxy never leaks to siblings
+  afterAll(() => {
+    if (original) setGlobalDispatcher(original);
+  });
+
+  it(
+    'resolves a real tiktok video (needs LIVE_PROXY; direct ip gets captcha)',
+    async (ctx) => {
+      if (!TT_ENABLED) {
+        ctx.skip();
+        return;
+      }
+      expect(TT_URL, 'no tiktok url in fixtures').toBeTruthy();
+      const info = await tiktok.getInfo(TT_URL, {});
+
+      expect(
+        info,
+        'null — tiktok served captcha (ip flagged, try a proxy)'
+      ).toBeTruthy();
+      expect(info?.title, 'no title resolved').toBeTruthy();
+      // canary for tiktok page changes
+      expect(
+        info?.formats?.length ?? 0,
+        'no formats — tiktok changed or captcha'
+      ).toBeGreaterThan(0);
+
+      console.log(
+        `[live] tiktok OK${PROXY ? ' (via proxy)' : ''}: "${info?.title}" — ${info?.formats?.length} format(s)`
+      );
+    },
+    60000
+  );
+});

backend/tests/manual/x_live.test.ts

@@ -0,0 +1,42 @@
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'node:fs';
+import { x } from '../../src/services/extractors/index.js';
+
+// live test, real x fetch
+// run on residential ip not datacenter
+const RUN = process.env.LIVE_TEST === '1';
+const ldescribe = RUN ? describe : describe.skip;
+
+interface LiveUrlEntry {
+  url: string;
+  note?: string;
+}
+
+// volatile urls live in one editable json
+const liveUrls = JSON.parse(
+  readFileSync(
+    new URL('../fixtures/live-extractor-urls.json', import.meta.url),
+    'utf8'
+  )
+) as Record<string, LiveUrlEntry>;
+
+const X_URL = process.env.X_LIVE_URL || liveUrls.x?.url;
+
+ldescribe('x extractor (live)', () => {
+  it('resolves a real x video to playable formats', async () => {
+    expect(X_URL, 'no x url in fixtures').toBeTruthy();
+    const info = await x.getInfo(X_URL, {});
+
+    expect(info, 'extractor returned null — likely broken').toBeTruthy();
+    expect(info?.title, 'no title resolved').toBeTruthy();
+    // canary for x page changes
+    expect(
+      info?.formats?.length ?? 0,
+      'no formats — x likely changed, extractor needs a fix'
+    ).toBeGreaterThan(0);
+
+    console.log(
+      `[live] x OK: "${info?.title}" — ${info?.formats?.length} format(s)`
+    );
+  }, 60000);
+});

backend/tests/setup.ts

@@ -453,41 +453,78 @@ export const handlers = [
       });
     return new HttpResponse('OK');
   }),
-  http.get('https://www.instagram.com/reel/DFQe23tOWKz/', (req) => {
-    const url = new URL(req.request.url);
-    if (url.searchParams.get('__a') === '1') {
-      return HttpResponse.json({
-        items: [
-          {
-            pk: '123456',
-            id: '123456',
-            caption: { text: 'Test Title #awesome' },
-            user: { full_name: 'Test User', username: 'test_user' },
-            image_versions2: { candidates: [{ url: 'https://thumb.jpg' }] },
-            video_versions: [
+  // oembed maps shortcode -> media id
+  http.get('https://i.instagram.com/api/v1/oembed/', () => {
+    return HttpResponse.json({ media_id: '123456_789' });
+  }),
+  // mobile private api, primary path
+  http.get('https://i.instagram.com/api/v1/media/:mediaId/info/', () => {
+    return HttpResponse.json({
+      items: [
+        {
+          code: 'DFQe23tOWKz',
+          pk: '123456',
+          caption: { text: 'Test Title #awesome' },
+          user: { full_name: 'Test User', username: 'test_user' },
+          image_versions2: {
+            candidates: [
               {
-                id: 'hd',
-                url: 'https://scontent.cdninstagram.com/v/test.mp4',
+                url: 'https://scontent.cdninstagram.com/thumb.jpg',
                 width: 1080,
                 height: 1920,
               },
             ],
           },
-        ],
-      });
-    }
-    return new HttpResponse(
-      '<html><body><script>window.__additionalDataLoaded(\'feed\', { "shortcode_media": { "video_url": "https://scontent.cdninstagram.com/v/test.mp4", "display_url": "https://thumb.jpg", "owner": { "username": "test_user" }, "edge_media_to_caption": { "edges": [{ "node": { "text": "Test Title #awesome" } }] } } });</script></body></html>',
-      { headers: { 'Content-Type': 'text/html' } }
-    );
+          video_versions: [
+            {
+              id: 'sd',
+              url: 'https://scontent.cdninstagram.com/v/test_sd.mp4',
+              width: 480,
+              height: 854,
+            },
+            {
+              id: 'hd',
+              url: 'https://scontent.cdninstagram.com/v/test.mp4',
+              width: 1080,
+              height: 1920,
+            },
+          ],
+        },
+      ],
+    });
   }),
-  http.get('https://api.instagram.com/oembed', () => {
+  // web graphql, secondary path
+  http.post('https://www.instagram.com/graphql/query', () => {
     return HttpResponse.json({
-      title: 'OEmbed Title',
-      author_name: 'OEmbed Author',
-      thumbnail_url: 'https://thumb.jpg',
+      data: {
+        xdt_shortcode_media: {
+          shortcode: 'DFQe23tOWKz',
+          video_url: 'https://scontent.cdninstagram.com/v/test.mp4',
+          display_url: 'https://scontent.cdninstagram.com/thumb.jpg',
+          is_video: true,
+          dimensions: { width: 1080, height: 1920 },
+          owner: { username: 'test_user', full_name: 'Test User' },
+          edge_media_to_caption: {
+            edges: [{ node: { text: 'Test Title #awesome' } }],
+          },
+        },
+      },
     });
   }),
+  // captioned embed, last resort
+  http.get(
+    'https://www.instagram.com/reel/DFQe23tOWKz/embed/captioned/',
+    () => {
+      return new HttpResponse(
+        '<html><head><meta property="og:title" content="Test Title #awesome"></head><body><script>{"video_url":"https://scontent.cdninstagram.com/v/test.mp4"}</script></body></html>',
+        { headers: { 'Content-Type': 'text/html' } }
+      );
+    }
+  ),
+  // size probe for cdn assets
+  http.head('https://scontent.cdninstagram.com/*', () => {
+    return new HttpResponse(null, { headers: { 'content-length': '1048576' } });
+  }),
 ];
 
 export const server = setupServer(...handlers);

backend/vitest.config.js

@@ -8,6 +8,8 @@ export default defineConfig({
   test: {
     globals: true,
     environment: 'node',
+    // vitest4 stopped auto-setting this; guards app boot
+    env: { NODE_ENV: 'test' },
     setupFiles: ['./tests/setup.ts'],
     testTimeout: 60000,
     reporters: ['default', 'junit'],