import './instrument.js'; import 'dotenv/config'; import dns from 'node:dns'; import { startCipherRotation } from './utils/network/cipher.util.js'; startCipherRotation(); import express, { Request, Response, NextFunction } from 'express'; import compression from 'compression'; import helmet from 'helmet'; import { rateLimit } from 'express-rate-limit'; import * as Sentry from '@sentry/node'; // skipcq: JS-C1003 import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { traceContext } from './utils/infra/trace.util.js'; import { randomUUID } from 'node:crypto'; import db from './utils/infra/db.util.js'; import videoRoutes from './routes/video.routes.js'; import keyChangerRoutes from './routes/keychanger.routes.js'; import remixRoutes from './routes/remix.routes.js'; import { requireApiKey, requireLocalOrApiKey, assertProdConfig, } from './utils/network/auth.util.js'; import { logger } from './utils/infra/logger.util.js'; import { setupGracefulShutdown } from './utils/infra/shutdown.util.js'; import { configureServerTimeouts } from './utils/infra/server-timeouts.util.js'; import { closeAllRedis } from './utils/infra/redis.util.js'; import { startJanitor } from './utils/infra/janitor.util.js'; import { warmUp } from './utils/infra/warmup.util.js'; import { metricsMiddleware, getMetrics, recordFailure, } from './utils/infra/metrics.util.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const STEMS_BASE_DIR = path.join(__dirname, '../temp/remix_stems'); // termux bypass if (process.platform === 'android') { try { const { Module } = await import('module'); const originalRequire = Module.prototype.require; Module.prototype.require = function ( this: unknown, name: string, ...args: unknown[] ) { if (name === '@ffmpeg-installer/ffmpeg') { return { path: 'ffmpeg', version: 'system', url: 'https://ffmpeg.org/', }; } if (name.includes('libsql')) { try { return ( originalRequire as (...innerArgs: unknown[]) => unknown ).apply(this, [name, ...args]); } catch (_ERROR) { console.debug('[System] LibSQL bypass error:', _ERROR); console.warn( `[System] LibSQL native library '${name}' not found, bypassing...` ); return { createClient: () => ({ execute: () => Promise.resolve({ rows: [] }), batch: () => Promise.resolve([]), close: () => {}, }), }; } } if (name === 'msgpackr-extract' || name === 'cpu-features') { return null; } return (originalRequire as (...innerArgs: unknown[]) => unknown).apply( this, [name, ...args] ); }; console.log('[System] Mocked native modules for Termux compatibility'); } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); console.warn( `[System] Failed to mock @ffmpeg-installer/ffmpeg: ${message}` ); } } const dnsModule = dns as unknown as { setDefaultResultOrder?: (order: 'ipv4first' | 'ipv6first') => void; }; if (dnsModule.setDefaultResultOrder) { dnsModule.setDefaultResultOrder('ipv4first'); } // global errors process.on('unhandledRejection', (reason: unknown) => { const message = reason instanceof Error ? reason.message : String(reason); console.error(`[Unhandled] reason: ${message}`); }); process.on('uncaughtException', (err: unknown) => { const message = err instanceof Error ? err.message : String(err); console.error(`[Uncaught] error: ${message}`); if (err instanceof Error && err.stack) console.error(err.stack); }); const app = express(); app.set('trust proxy', 1); // observe latency + outcomes app.use(metricsMiddleware); const PORT = Number(process.env.PORT) || 5000; app.use( helmet({ crossOriginResourcePolicy: { policy: 'cross-origin' }, contentSecurityPolicy: { directives: { defaultSrc: ["'self'"], // no inline scripts; external bundles only scriptSrc: ["'self'"], // styles still inline (react/libs); hardened later styleSrc: ["'self'", "'unsafe-inline'"], imgSrc: ["'self'", 'data:', 'https:'], connectSrc: ["'self'", 'https:', 'wss:'], }, }, }) ); const globalLimiter = rateLimit({ windowMs: 15 * 60 * 1000, max: 100, standardHeaders: true, legacyHeaders: false, message: { error: 'Too many requests, please try again later.' }, }); app.use('/api/', globalLimiter); const infoLimiter = rateLimit({ windowMs: 1 * 60 * 1000, max: 15, standardHeaders: true, legacyHeaders: false, message: { error: 'Extraction rate limit exceeded. Slow down!' }, }); app.use(['/info', '/stream-urls'], infoLimiter); // skip HEAD to allow resumes const convertLimiter = rateLimit({ windowMs: 5 * 60 * 1000, max: 60, standardHeaders: true, legacyHeaders: false, skip: (req: Request) => req.method === 'HEAD', message: { error: 'Download rate limit exceeded. Slow down!' }, }); app.use('/convert', convertLimiter); // seeder is expensive to run const seedLimiter = rateLimit({ windowMs: 15 * 60 * 1000, max: 10, standardHeaders: true, legacyHeaders: false, message: { error: 'Seeder rate limit exceeded.' }, }); app.use('/seed-intelligence', seedLimiter); // disable SSE compression app.use( compression({ filter: (req, res) => { if (req.path === '/events') return false; if (res.getHeader('Content-Type') === 'text/event-stream') return false; return compression.filter(req, res); }, }) ); app.use((req: Request, res: Response, next: NextFunction) => { const traceId = (req.headers['x-correlation-id'] as string) || (req.query.id as string) || randomUUID().split('-')[0]; res.setHeader('X-Trace-Id', traceId); Sentry.withIsolationScope((scope) => { if (process.env.SENTRY_DSN) { scope.setTag('traceId', traceId); } traceContext.run({ traceId }, () => { next(); }); }); }); app.use((req: Request, res: Response, next: NextFunction) => { if (req.path === '/ping' || req.method === 'OPTIONS') { next(); return; } const timestamp = new Date().toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit', }); const traceId = (traceContext.getStore() as { traceId?: string })?.traceId || 'global'; console.log( `[${timestamp}] [${traceId}] ${req.method} ${req.originalUrl || req.url}` ); next(); }); // cors middleware app.use((req: Request, res: Response, next: NextFunction) => { const origin = req.headers.origin as string | undefined; const allowlist = (process.env.ALLOWED_ORIGINS || '') .split(',') .map((entry) => entry.trim()) .filter(Boolean); if (origin) { const openMode = allowlist.length === 0; if (openMode || allowlist.includes(origin)) { res.header('Access-Control-Allow-Origin', origin); res.header('Vary', 'Origin'); // credentials unsafe with reflected origin if (!openMode) res.header('Access-Control-Allow-Credentials', 'true'); } } res.header( 'Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS, PATCH' ); res.header( 'Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept, Authorization, Cache-Control, Last-Event-ID, ngrok-skip-browser-warning, bypass-tunnel-reminder, sentry-trace, baggage' ); // eme worker reads these for progress res.header( 'Access-Control-Expose-Headers', 'Content-Length, Content-Range, Accept-Ranges' ); if (req.method === 'OPTIONS') { res.status(200).end(); return; } next(); }); console.log('--- Environment Check ---'); console.log(`PORT: ${PORT}`); console.log( `COOKIES_URL: ${process.env.COOKIES_URL ? '✅ LOADED' : '❌ MISSING'}` ); console.log( `GEMINI_API_KEY: ${process.env.GEMINI_API_KEY ? '✅ LOADED' : '❌ MISSING'}` ); console.log( `GROQ_API_KEY: ${process.env.GROQ_API_KEY ? '✅ LOADED' : '❌ MISSING'}` ); console.log( `BILIBILI_COOKIE: ${process.env.BILIBILI_COOKIE ? '✅ LOADED' : '➖ not set (720p cap)'}` ); console.log( `INFO CACHE: ${process.env.DISABLE_INFO_CACHE && process.env.DISABLE_INFO_CACHE !== '0' ? '🚫 DISABLED (testing)' : '✅ ENABLED'}` ); dns.lookup('google.com', { family: 4 }, (err, addr) => { const status = err ? '❌ FAILED' : `✅ ${addr}`; console.log(`DNS google.com: ${status}`); }); dns.lookup('youtube.com', { family: 4 }, (err, addr) => { const status = err ? '❌ FAILED' : `✅ ${addr}`; console.log(`DNS youtube.com: ${status}`); }); console.log('-------------------------'); app.use((_req: Request, res: Response, next: NextFunction) => { res.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); res.setHeader('Cross-Origin-Embedder-Policy', 'credentialless'); res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); next(); }); app.use(express.json({ limit: '1mb' })); app.use(express.urlencoded({ limit: '1mb', extended: true })); // core routes const TEMP_DIR = path.join(__dirname, 'temp'); const CACHE_DIR = path.join(TEMP_DIR, 'yt-dlp-cache'); [TEMP_DIR, CACHE_DIR].forEach((dir) => { if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } }); console.log('[System] Initializing routes...'); app.get('/ping', (_req: Request, res: Response) => { res.status(200).send('pong'); }); app.get('/api/get-url', async (_req: Request, res: Response) => { try { if (db) { const result = (await db.execute({ sql: "SELECT value FROM configs WHERE key = 'BACKEND_URL' LIMIT 1", args: [], })) as unknown as { rows: Array<{ value: string }> }; if (result.rows.length > 0) { res.json({ url: result.rows[0].value }); return; } } } catch (error) { console.error('[Discovery] Error fetching URL:', error); } res.json({ url: null }); }); // opt-in auth; off unless API_KEY set app.use( [ '/info', '/stream-urls', '/convert', '/proxy', '/api/remix', '/api/key-changer', ], requireApiKey ); app.use('/', videoRoutes); app.use('/api/key-changer', keyChangerRoutes); app.use('/api/remix', remixRoutes); console.log('[System] Routes ready'); if (process.env.SENTRY_DSN) { Sentry.setupExpressErrorHandler(app); } app.get('/health', (_req: Request, res: Response) => { res.status(200).json({ status: 'ok', port: PORT, }); }); // gated: localhost or API_KEY only app.get('/metrics', requireLocalOrApiKey, (_req: Request, res: Response) => { res.status(200).json(getMetrics()); }); // global error handler app.use((err: unknown, req: Request, res: Response, _next: NextFunction) => { const reason = err instanceof Error ? err.name : 'UnknownError'; recordFailure(reason); logger.error({ err, path: req.path, method: req.method }, 'request error'); if (!res.headersSent) { // hide internals from clients in production const details = process.env.NODE_ENV === 'production' ? undefined : err instanceof Error ? err.message : typeof err === 'string' ? err : JSON.stringify(err); res.status(500).json({ error: 'Internal Server Error', details }); } }); const distPath = path.join(__dirname, '../../frontend/dist'); if (fs.existsSync(distPath) && process.env.API_ONLY !== 'true') { app.use(express.static(distPath)); app.get(/.*/u, (req: Request, res: Response, next: NextFunction) => { if ( req.path.startsWith('/events') || req.path.startsWith('/info') || req.path.startsWith('/convert') || req.path.startsWith('/stream-urls') || req.path.startsWith('/proxy') || req.path.startsWith('/api') || req.path.includes('/EME_STREAM_DOWNLOAD/') ) { next(); return; } res.sendFile(path.join(distPath, 'index.html')); }); } else { app.get('/', (_req: Request, res: Response) => { res.send('YouTube to MP4 Backend is running!'); }); } export default app; if (process.env.NODE_ENV !== 'test') { assertProdConfig(); const server = app.listen(PORT, '0.0.0.0', () => { console.log(`Server is running on port ${PORT}`); configureServerTimeouts(server); setupGracefulShutdown(server, { onClose: closeAllRedis }); import('./services/ytdlp/info.js').then(({ startPeerKeepWarm }) => startPeerKeepWarm() ); import('./services/ytdlp/config.js').then(({ bootstrapCookies }) => bootstrapCookies() ); import('node:child_process').then(({ exec, spawn: spawnChild }) => { exec('yt-dlp --version', (err, stdout) => { if (err) console.error(`yt-dlp check failed: ${err.message}`); else console.log(`yt-dlp: ${stdout.trim()}`); }); exec('ffmpeg -version', (err, stdout) => { if (err) console.error(`FFmpeg check failed: ${err.message}`); else console.log(`FFmpeg: ${stdout.split('\n')[0]}`); }); // pot opt-in; bgutil currently fails botguard const potEnabled = process.env.ENABLE_POT_PLUGIN === '1'; if (!potEnabled) { console.log('[PO Token] disabled; set ENABLE_POT_PLUGIN=1 to enable'); } else { const potCandidates = [ process.env.HOME ? path.resolve( process.env.HOME, 'bgutil-ytdlp-pot-provider/server/build/main.js' ) : null, '/root/bgutil-ytdlp-pot-provider/server/build/main.js', '/data/data/com.termux/files/home/bgutil-ytdlp-pot-provider/server/build/main.js', ].filter((candidate): candidate is string => Boolean(candidate)); try { const potScript = potCandidates.find((candidate) => fs.existsSync(candidate) ); if (potScript) { const pot = spawnChild('node', [potScript], { stdio: 'ignore', detached: true, }); pot.unref(); console.log( `PO Token server started (pid: ${pot.pid}, port: 4416)` ); } else { console.log( `[PO Token] Server script not found in: ${potCandidates.join(', ')}` ); } } catch (error: unknown) { console.error('[PO Token] Spawn failed:', (error as Error).message); } } }); warmUp(); // log memory levers for resource tuning setTimeout(() => { const mem = process.memoryUsage(); const mb = (bytes: number) => Math.round(bytes / 1048576); const limit = process.env.KOYEB_INSTANCE_MEMORY_MB || '?'; console.log( `[Mem] idle snapshot: rss=${mb(mem.rss)}MB heapUsed=${mb(mem.heapUsed)}MB ` + `heapTotal=${mb(mem.heapTotal)}MB external=${mb(mem.external)}MB ` + `arrayBuffers=${mb(mem.arrayBuffers)}MB (instance limit=${limit}MB)` ); }, 6000).unref(); }); } startJanitor({ tempDir: TEMP_DIR, stemsBaseDir: STEMS_BASE_DIR, db });