Spaces:
Paused
Paused
circleci commited on
Commit ·
dae0a8c
1
Parent(s): 2fee6e6
deploy beb5a90
Browse files- backend/Dockerfile +3 -3
- backend/README.md +4 -4
- backend/ci-results/security-scan-tests.json +4 -0
- backend/ci-results/test-backend-test-results.xml +0 -0
- backend/ci-results/test-backend-tests.json +0 -0
- backend/ci-results/test-frontend-tests.json +4 -0
- backend/eslint.config.js +4 -4
- backend/facebook_cookies.txt +11 -0
- backend/src/services/extractors/instagram/fetcher.ts +2 -2
- backend/src/services/extractors/instagram/index.ts +1 -1
- backend/src/services/ytdlp/info-youtube.ts +1 -1
- backend/temp_cookies.txt +26 -0
- backend/youtube_cookies.txt +15 -0
backend/Dockerfile
CHANGED
|
@@ -27,14 +27,14 @@ ENV PATH="/opt/tool-venv/bin:$PATH"
|
|
| 27 |
|
| 28 |
# shared has its own deps (zod) the schemas need
|
| 29 |
WORKDIR /app/shared
|
| 30 |
-
COPY shared/ ./
|
| 31 |
RUN npm ci
|
| 32 |
|
| 33 |
WORKDIR /app/backend
|
| 34 |
-
COPY backend/package*.json ./
|
| 35 |
RUN npm ci
|
| 36 |
|
| 37 |
-
COPY backend/ ./
|
| 38 |
RUN npx tsc
|
| 39 |
|
| 40 |
RUN mkdir -p temp && chmod 777 temp
|
|
|
|
| 27 |
|
| 28 |
# shared has its own deps (zod) the schemas need
|
| 29 |
WORKDIR /app/shared
|
| 30 |
+
COPY web/shared/ ./
|
| 31 |
RUN npm ci
|
| 32 |
|
| 33 |
WORKDIR /app/backend
|
| 34 |
+
COPY web/backend/package*.json ./
|
| 35 |
RUN npm ci
|
| 36 |
|
| 37 |
+
COPY web/backend/ ./
|
| 38 |
RUN npx tsc
|
| 39 |
|
| 40 |
RUN mkdir -p temp && chmod 777 temp
|
backend/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# Backend
|
| 2 |
|
| 3 |
-
the Express 5 + TypeScript service that powers stream resolution, muxing, the Spotify resolution race, and the Remix Lab API. for the project overview see [`../README.md`](../README.md). for self-hosting see [`../docs/run-an-instance.md`](../docs/run-an-instance.md).
|
| 4 |
|
| 5 |
## Layout
|
| 6 |
|
|
@@ -53,11 +53,11 @@ backend/
|
|
| 53 |
|
| 54 |
`remix.routes.ts` proxies the Python Remix Lab kernel and serves stem/chord/beat results. `keychanger.routes.ts` handles pitch-shift uploads. `/convert` and `/proxy` are gated by `concurrencyGuard(2)` (`utils/network/security.util.ts`) to keep memory bounded on Termux and free-tier hosts.
|
| 55 |
|
| 56 |
-
response shapes for `/info` and `/convert` are documented in [`../docs/api.md`](../docs/api.md).
|
| 57 |
|
| 58 |
## Environment
|
| 59 |
|
| 60 |
-
configure via `backend/.env`. the full reference is in [`../docs/env-variables.md`](../docs/env-variables.md). the backend boots without most of these — they enable optional features and degrade gracefully when unset. minimum to get something useful:
|
| 61 |
|
| 62 |
- `REDIS_URL` — Redis for the metadata cache and BullMQ job queue (defaults to `redis://127.0.0.1:6379`).
|
| 63 |
- `GEMINI_API_KEY` and/or `GROQ_API_KEY` — at least one for the AI fallback in Spotify resolution.
|
|
@@ -97,4 +97,4 @@ other scripts:
|
|
| 97 |
- **MP4 finalization**: `-movflags +faststart` is set on finalized MP4 downloads so playback can start before the file finishes.
|
| 98 |
- **SSE**: `/events` is the canonical telemetry channel. resolvers, downloaders, and the seeder push progress through `utils/network/sse.util.ts`.
|
| 99 |
|
| 100 |
-
before putting an instance on the public internet, read [`../docs/protect-an-instance.md`](../docs/protect-an-instance.md).
|
|
|
|
| 1 |
# Backend
|
| 2 |
|
| 3 |
+
the Express 5 + TypeScript service that powers stream resolution, muxing, the Spotify resolution race, and the Remix Lab API. for the project overview see [`../../README.md`](../../README.md). for self-hosting see [`../../docs/run-an-instance.md`](../../docs/run-an-instance.md).
|
| 4 |
|
| 5 |
## Layout
|
| 6 |
|
|
|
|
| 53 |
|
| 54 |
`remix.routes.ts` proxies the Python Remix Lab kernel and serves stem/chord/beat results. `keychanger.routes.ts` handles pitch-shift uploads. `/convert` and `/proxy` are gated by `concurrencyGuard(2)` (`utils/network/security.util.ts`) to keep memory bounded on Termux and free-tier hosts.
|
| 55 |
|
| 56 |
+
response shapes for `/info` and `/convert` are documented in [`../../docs/api.md`](../../docs/api.md).
|
| 57 |
|
| 58 |
## Environment
|
| 59 |
|
| 60 |
+
configure via `web/backend/.env`. the full reference is in [`../../docs/env-variables.md`](../../docs/env-variables.md). the backend boots without most of these — they enable optional features and degrade gracefully when unset. minimum to get something useful:
|
| 61 |
|
| 62 |
- `REDIS_URL` — Redis for the metadata cache and BullMQ job queue (defaults to `redis://127.0.0.1:6379`).
|
| 63 |
- `GEMINI_API_KEY` and/or `GROQ_API_KEY` — at least one for the AI fallback in Spotify resolution.
|
|
|
|
| 97 |
- **MP4 finalization**: `-movflags +faststart` is set on finalized MP4 downloads so playback can start before the file finishes.
|
| 98 |
- **SSE**: `/events` is the canonical telemetry channel. resolvers, downloaders, and the seeder push progress through `utils/network/sse.util.ts`.
|
| 99 |
|
| 100 |
+
before putting an instance on the public internet, read [`../../docs/protect-an-instance.md`](../../docs/protect-an-instance.md).
|
backend/ci-results/security-scan-tests.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"items": [],
|
| 3 |
+
"next_page_token": null
|
| 4 |
+
}
|
backend/ci-results/test-backend-test-results.xml
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
backend/ci-results/test-backend-tests.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
backend/ci-results/test-frontend-tests.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"items": [],
|
| 3 |
+
"next_page_token": null
|
| 4 |
+
}
|
backend/eslint.config.js
CHANGED
|
@@ -4,13 +4,13 @@ import tseslint from 'typescript-eslint';
|
|
| 4 |
import sonarjs from 'eslint-plugin-sonarjs';
|
| 5 |
import prettierConfig from 'eslint-config-prettier';
|
| 6 |
import { existsSync } from 'node:fs';
|
| 7 |
-
import { join } from 'node:path';
|
| 8 |
-
import
|
| 9 |
|
| 10 |
-
const pluginPath = join(
|
| 11 |
const hasPlugin = existsSync(pluginPath);
|
| 12 |
const nexstreamPlugin = hasPlugin
|
| 13 |
-
? (await import('../scripts/eslint-plugin-nexstream.js')).default
|
| 14 |
: null;
|
| 15 |
|
| 16 |
export default tseslint.config(
|
|
|
|
| 4 |
import sonarjs from 'eslint-plugin-sonarjs';
|
| 5 |
import prettierConfig from 'eslint-config-prettier';
|
| 6 |
import { existsSync } from 'node:fs';
|
| 7 |
+
import { join, dirname } from 'node:path';
|
| 8 |
+
import { fileURLToPath } from 'node:url';
|
| 9 |
|
| 10 |
+
const pluginPath = join(dirname(fileURLToPath(import.meta.url)), '../../scripts/eslint-plugin-nexstream.js');
|
| 11 |
const hasPlugin = existsSync(pluginPath);
|
| 12 |
const nexstreamPlugin = hasPlugin
|
| 13 |
+
? (await import('../../scripts/eslint-plugin-nexstream.js')).default
|
| 14 |
: null;
|
| 15 |
|
| 16 |
export default tseslint.config(
|
backend/facebook_cookies.txt
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Netscape HTTP Cookie File
|
| 2 |
+
# This file is generated by yt-dlp. Do not edit.
|
| 3 |
+
|
| 4 |
+
.facebook.com TRUE / TRUE 1784546865 datr 5WD3Z2AfXZVSxZCH4yapnyD9
|
| 5 |
+
.facebook.com TRUE / TRUE 1805180241 sb 5WD3Z1jSQy5EgJXUVh5vgb89
|
| 6 |
+
.facebook.com TRUE / TRUE 1779253025 ps_l 1
|
| 7 |
+
.facebook.com TRUE / TRUE 1779253025 ps_n 1
|
| 8 |
+
.facebook.com TRUE / TRUE 1808801368 c_user 61558851086624
|
| 9 |
+
.facebook.com TRUE / TRUE 1785041368 fr 19UDONV88XhKGe2RF.AWf4M4X9z4Ngaor9pvZ5MbZyPg_kHIz-h5WV6izpvU192apVcUI.Bp7urX..AAA.0.0.Bp7urX.AWeV8ErxwjQi2vTc5yCctfMZQGE
|
| 10 |
+
.facebook.com TRUE / TRUE 1808801368 xs 45%3Abib84KIB5WBKnA%3A2%3A1770620237%3A-1%3A-1%3A%3AAcwTxaZyeBZFm282q3eAouldL84OxiuyfhsHaO40ug
|
| 11 |
+
.facebook.com TRUE / TRUE 0 presence C%7B%22t3%22%3A%5B%5D%2C%22utc3%22%3A1777265373434%2C%22v%22%3A1%7D
|
backend/src/services/extractors/instagram/fetcher.ts
CHANGED
|
@@ -106,7 +106,7 @@ export function shortcodeToMediaId(shortcode: string): string {
|
|
| 106 |
return pk.toString();
|
| 107 |
}
|
| 108 |
|
| 109 |
-
// Set-Cookie -> replayable header (anonymous csrftoken/mid, NOT login cookie)
|
| 110 |
function jarFromResponse(
|
| 111 |
response: globalThis.Response
|
| 112 |
): Record<string, string> {
|
|
@@ -184,7 +184,7 @@ async function getSession(
|
|
| 184 |
}
|
| 185 |
|
| 186 |
// logged-out post resolve: reuse cached page tokens, then /api/graphql.
|
| 187 |
-
// IG gates old shortcode query behind auth, but this media_id query is
|
| 188 |
// served logged-out — sec-fetch headers keep IG returning JSON not html shell
|
| 189 |
export async function fetchLoggedOutMedia(
|
| 190 |
shortcode: string,
|
|
|
|
| 106 |
return pk.toString();
|
| 107 |
}
|
| 108 |
|
| 109 |
+
// Set-Cookie -> replayable header (anonymous csrftoken/mid, NOT login cookie) // eslint-disable-line nexstream/nexstream-comments
|
| 110 |
function jarFromResponse(
|
| 111 |
response: globalThis.Response
|
| 112 |
): Record<string, string> {
|
|
|
|
| 184 |
}
|
| 185 |
|
| 186 |
// logged-out post resolve: reuse cached page tokens, then /api/graphql.
|
| 187 |
+
// IG gates old shortcode query behind auth, but this media_id query is // eslint-disable-line nexstream/nexstream-comments
|
| 188 |
// served logged-out — sec-fetch headers keep IG returning JSON not html shell
|
| 189 |
export async function fetchLoggedOutMedia(
|
| 190 |
shortcode: string,
|
backend/src/services/extractors/instagram/index.ts
CHANGED
|
@@ -45,7 +45,7 @@ async function resolveParsed(
|
|
| 45 |
const parsed = await resolve();
|
| 46 |
if (parsed && parsed.media.length > 0) return parsed;
|
| 47 |
} catch (error: unknown) {
|
| 48 |
-
// IG rate-limiting — stop cascade rather than hammering every path
|
| 49 |
if ((error as { retryable?: boolean })?.retryable) throw error;
|
| 50 |
console.debug(`[JS-IG] path failed: ${(error as Error).message}`);
|
| 51 |
}
|
|
|
|
| 45 |
const parsed = await resolve();
|
| 46 |
if (parsed && parsed.media.length > 0) return parsed;
|
| 47 |
} catch (error: unknown) {
|
| 48 |
+
// IG rate-limiting — stop cascade rather than hammering every path // eslint-disable-line nexstream/nexstream-comments
|
| 49 |
if ((error as { retryable?: boolean })?.retryable) throw error;
|
| 50 |
console.debug(`[JS-IG] path failed: ${(error as Error).message}`);
|
| 51 |
}
|
backend/src/services/ytdlp/info-youtube.ts
CHANGED
|
@@ -126,7 +126,7 @@ export async function handleYoutubeTiktokInfo(
|
|
| 126 |
};
|
| 127 |
await setCachedInfo(cacheKey, fullInfo);
|
| 128 |
|
| 129 |
-
// JS already returned HD; only enhance via yt-dlp when opted in
|
| 130 |
if (!isTikTok && process.env.YT_DLP_ENHANCE === '1')
|
| 131 |
void runYtdlpEnhancement(
|
| 132 |
cacheKey,
|
|
|
|
| 126 |
};
|
| 127 |
await setCachedInfo(cacheKey, fullInfo);
|
| 128 |
|
| 129 |
+
// JS already returned HD; only enhance via yt-dlp when opted in // eslint-disable-line nexstream/nexstream-comments
|
| 130 |
if (!isTikTok && process.env.YT_DLP_ENHANCE === '1')
|
| 131 |
void runYtdlpEnhancement(
|
| 132 |
cacheKey,
|
backend/temp_cookies.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Netscape HTTP Cookie File
|
| 2 |
+
# This file is generated by yt-dlp. Do not edit.
|
| 3 |
+
|
| 4 |
+
.youtube.com TRUE / FALSE 0 PREF f6=40000000&tz=UTC&f7=100&f4=4000000&hl=en
|
| 5 |
+
.youtube.com TRUE / FALSE 1802593048 HSID AMY9owAf1zjps8RNZ
|
| 6 |
+
.youtube.com TRUE / TRUE 1802593048 SSID Al0WBDo16MQQN0E62
|
| 7 |
+
.youtube.com TRUE / FALSE 1802593048 APISID -6sBzQS5APgo0h2H/Afd_pLBBCBr7BtEni
|
| 8 |
+
.youtube.com TRUE / TRUE 1802593048 SAPISID QKYsvlhchM0hAEvv/Av0Bn7_pR3oGo4ksY
|
| 9 |
+
.youtube.com TRUE / TRUE 1802593048 __Secure-1PAPISID QKYsvlhchM0hAEvv/Av0Bn7_pR3oGo4ksY
|
| 10 |
+
.youtube.com TRUE / TRUE 1802593048 __Secure-3PAPISID QKYsvlhchM0hAEvv/Av0Bn7_pR3oGo4ksY
|
| 11 |
+
.youtube.com TRUE / FALSE 1802593048 SID g.a0005ghxRHGB_w-Qxm9zoJuKGR4ftChQ5g3l-mAROSkNtWaCtmDsUxQkqwHEC6_3ZyDTAFYpKAACgYKAY4SARcSFQHGX2MiOPuh0Fc1utZateAkAiVxwhoVAUF8yKrcFGutbai4YSuMtaKSYLKC0076
|
| 12 |
+
.youtube.com TRUE / TRUE 1802593048 __Secure-1PSID g.a0005ghxRHGB_w-Qxm9zoJuKGR4ftChQ5g3l-mAROSkNtWaCtmDsTNI-N1qsAsxtQ64wQq3nHgACgYKAVkSARcSFQHGX2MiJjRw70uN3fX8dMxG6dcz2xoVAUF8yKpBbIAXR7NCncXL_zgM7JFS0076
|
| 13 |
+
.youtube.com TRUE / TRUE 1802593048 __Secure-3PSID g.a0005ghxRHGB_w-Qxm9zoJuKGR4ftChQ5g3l-mAROSkNtWaCtmDssC8jNY6aRqqtU956ISzcVwACgYKAcUSARcSFQHGX2Mi6Z6gOSXcF6VdmU0exBgVvBoVAUF8yKqdEewjtu72m0dkdI7Bb3b20076
|
| 14 |
+
.youtube.com TRUE / TRUE 1802593049 LOGIN_INFO AFmmF2swRgIhAM396oCYxY_Gt8eWWvTIPaxkddQw9kqmxVv9iRZto3P5AiEAuBV0gMFQgizqziQ9NrGNKOSNG6Un0sCdV7Wf7NBY6JI:QUQ3MjNmeVlmS192dk1mWHNEdFB4TjhTcjJ0Sm5oWlZtMFBHdDgwZmZNZEpBV1IzQzRkTVpHZ3o1ZkMyMEJSR01zM3dTRVEycnNveEdscWcxS0VsUkxNRl9wek1LUGpyWXQ2UEpCcFZldG02VkhKcENHZlFHNzB4WFlPNk84akdoVk1pWVFVMGh6em1QbHR1MFBWckZGQ25xYXh1VE54SW93
|
| 15 |
+
.youtube.com TRUE / TRUE 1802022626 __Secure-1PSIDTS sidts-CjUB7I_69Gg5PPVJMY3ZmUw7MbeMoHvwd4AgOfyBuTwf-oUo0etiGiy9dfmi3cJZG3QDI4gDWhAA
|
| 16 |
+
.youtube.com TRUE / TRUE 1802022626 __Secure-3PSIDTS sidts-CjUB7I_69Gg5PPVJMY3ZmUw7MbeMoHvwd4AgOfyBuTwf-oUo0etiGiy9dfmi3cJZG3QDI4gDWhAA
|
| 17 |
+
.youtube.com TRUE / FALSE 1802104616 SIDCC AKEyXzW6rJpfJu1AAtIQa3UnncmZlJQz2FgPDJ2JWb37lI2o67HxxSKlXljqIaJE1eaWkoKB
|
| 18 |
+
.youtube.com TRUE / TRUE 1802104616 __Secure-1PSIDCC AKEyXzX6fTh1uav-l_51ixvEJlc6U2Kh01Bj5ovhYia6JxWJ1t-0ckJEM0CBXRN4GnryOx0NTA
|
| 19 |
+
.youtube.com TRUE / TRUE 1802104616 __Secure-3PSIDCC AKEyXzW6JYTeluTH5xCTcjo26uOqyKcHZfnfXUCQefrB0lcAyF6GXvc9ukP4P8Eu1eFuNuw1Yw
|
| 20 |
+
.youtube.com TRUE / TRUE 0 YSC bqf6NUS4Q6g
|
| 21 |
+
.youtube.com TRUE / TRUE 1786120616 VISITOR_INFO1_LIVE dp3e_nSe7tU
|
| 22 |
+
.youtube.com TRUE / TRUE 1786120616 VISITOR_PRIVACY_METADATA CgJQSBIEGgAgDw%3D%3D
|
| 23 |
+
.youtube.com TRUE / TRUE 1786120215 __Secure-ROLLOUT_TOKEN CMWQ_MCe_Z25jgEQ86i-y6jKkgMY3tevzKjKkgM%3D
|
| 24 |
+
.youtube.com TRUE / TRUE 1833640616 __Secure-YT_TVFAS t=491824&s=2
|
| 25 |
+
.youtube.com TRUE / TRUE 1786120616 DEVICE_INFO ChxOell3TkRVek1qVTROemt6TVRjNU1ESXlPQT09EKj3oswGGJn0oswG
|
| 26 |
+
.youtube.com TRUE /tv TRUE 1803400616 __Secure-YT_DERP CPGatM2BBA%3D%3D
|
backend/youtube_cookies.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Netscape HTTP Cookie File
|
| 2 |
+
# This file is generated by yt-dlp. Do not edit.
|
| 3 |
+
|
| 4 |
+
.youtube.com TRUE / FALSE 0 PREF tz=UTC&f4=4000000&f7=100&f6=40000000&hl=en
|
| 5 |
+
.youtube.com TRUE / TRUE 1808440699 __Secure-3PAPISID y5pQqHVUZStFH2AW/AvSBkKTBRTnYjk4qH
|
| 6 |
+
.youtube.com TRUE / TRUE 1808440699 __Secure-3PSID g.a0007wh5iBXFaaHEI-Hz4oBHuJ6p0KdK3Nk96GklcE0wV-RlAySfnIV2i0xahOVmDgg10caxqQACgYKASESARASFQHGX2MidghmVUD0__2OGBxPY_UdthoVAUF8yKoCyoiqxb_wWpc1G8FhmjUb0076
|
| 7 |
+
.youtube.com TRUE / TRUE 1805831903 __Secure-1PSIDTS sidts-CjQBWhotCdNFAk0zmZcDTj_czZQglenZfRqtaUUENxqHxIOgRgTk_qIQZIwpWUcb0H947CewEAA
|
| 8 |
+
.youtube.com TRUE / TRUE 1805831903 __Secure-3PSIDTS sidts-CjQBWhotCdNFAk0zmZcDTj_czZQglenZfRqtaUUENxqHxIOgRgTk_qIQZIwpWUcb0H947CewEAA
|
| 9 |
+
.youtube.com TRUE / TRUE 1805831906 __Secure-3PSIDCC AKEyXzWYdD85_KP4QfHB0Mlo2dDFxE9sni9k-NN-tIGfwaHW6vk55jQVSZpsUvEOhft2kqgNbj8
|
| 10 |
+
.youtube.com TRUE / TRUE 1794591135 __Secure-YNID 18.YT=1YFLiGwm0cW5oCQ7iscXhoG5_9Tg27oIWqfmPKjyPKCG5hAWnT9uB2nkwRf75MghUBc8SdEsgZR8-wdRp4ZgXgVn_veGdJFeGKI5Jt6wdbIUql6GzveMD9NWNZa0dwEPmdQQwLB61GPMYJrt60i49TLSxC5gZE4xJXFE16i7GalsEgKGvxXjHxirWHJjOUpivWoTuhNfrXmnn6ECH0695OqrH7nopBRDF6XJN7OxLgaOj_eZxRNIoe7Pi_iSh-g6Nl0y_woTCSUWuHEQPa78pePgcHeRubtgwFO990s4So5O3760PeBrHiQxpcYkKmkop8Q3doceyJ5cyzLUjEf5jw
|
| 11 |
+
.youtube.com TRUE / TRUE 1779040935 GPS 1
|
| 12 |
+
.youtube.com TRUE / TRUE 0 YSC uUrb1X-RXqo
|
| 13 |
+
.youtube.com TRUE / TRUE 1794591135 VISITOR_INFO1_LIVE T67cF3c6mv0
|
| 14 |
+
.youtube.com TRUE / TRUE 1794591135 VISITOR_PRIVACY_METADATA CgJQSBIEGgAgYg%3D%3D
|
| 15 |
+
.youtube.com TRUE / TRUE 1794591135 __Secure-ROLLOUT_TOKEN COLw-NOisYO2VRDsmMWd7cCUAxjsmMWd7cCUAw%3D%3D
|