import { Cache, HEADERS_FOR_IP_FORWARDING, INTERNAL_SECRET_HEADER, Env, maskSensitiveInfo, } from './index.js'; import { config as appConfig } from '../config/index.js'; import { BodyInit, Dispatcher, fetch, Headers, HeadersInit, ProxyAgent, RequestInit, } from 'undici'; import { socksDispatcher } from 'fetch-socks'; import { createLogger } from '../logging/logger.js'; const logger = createLogger('http'); const urlCount = Cache.getInstance( 'url-count', undefined, 'memory' ); export class PossibleRecursiveRequestError extends Error { constructor(message: string) { super(message); this.name = 'PossibleRecursiveRequestError'; } } export function makeUrlLogSafe(url: string) { // for each component of the path, if it is longer than 10 characters, mask it // and replace the query params of key 'password' with '****' return url .split('/') .map((component) => { if (component.length > 10 && !component.includes('.')) { return maskSensitiveInfo(component); } return component; }) .join('/') .replace(/(? appConfig.recursion.thresholdLimit && !options.ignoreRecursion ) { logger.warn( { url: makeUrlLogSafe(urlObj.toString()), count: currentCount }, 'detected possible recursive requests, blocking' ); throw new PossibleRecursiveRequestError( `Possible recursive request to ${urlObj.toString()}` ); } if (currentCount > 0) { await urlCount.update(key, currentCount + 1); } else { await urlCount.set(key, 1, appConfig.recursion.thresholdWindow); } let dispatcher: Dispatcher | undefined; if (options.forceProxy) { dispatcher = getProxyAgent(options.forceProxy); } else if (useProxy) { dispatcher = getProxyAgent(appConfig.http.addonProxy[proxyIndex]); } logger.trace( { url: makeUrlLogSafe(urlObj.toString()), method: options.method ?? 'GET', proxy: useProxy ? `proxy-${proxyIndex + 1}` : options.forceProxy ? 'forced' : 'direct', }, 'http request' ); let response; try { response = await fetch(urlObj.toString(), { ...options.rawOptions, method: options.method, body: options.body, headers: headers, dispatcher: dispatcher, signal: AbortSignal.timeout(options.timeout), }); } catch (err) { if ( err instanceof Error && err.name === 'TypeError' && err.message === 'fetch failed' && err.cause ) { const cause = { ...(err.cause as Record) }; delete cause.stack; logger.error({ cause }, 'fetch failed due to network error'); } throw err; } return response; } const proxyAgents = new Map(); export function getProxyAgent(proxyUrl: string): Dispatcher | undefined { if (!proxyUrl) { return undefined; } let proxyAgent = proxyAgents.get(proxyUrl); if (!proxyAgent) { const proxyUrlObj = new URL(proxyUrl); if ( proxyUrlObj.protocol === 'socks5:' || proxyUrlObj.protocol === 'socks5h:' ) { proxyAgent = socksDispatcher({ type: 5, port: parseInt(proxyUrlObj.port), host: proxyUrlObj.hostname, userId: proxyUrlObj.username || undefined, password: proxyUrlObj.password || undefined, }); } else { proxyAgent = new ProxyAgent(proxyUrl); } } return proxyAgent; } export function shouldProxy(url: URL): { useProxy: boolean; proxyIndex: number; } { let useProxy = false; let hostname = url.hostname; let proxyIndex = -1; if (!appConfig.http.addonProxy || appConfig.http.addonProxy.length === 0) { return { useProxy: false, proxyIndex }; } if (hostname === 'localhost') { return { useProxy: false, proxyIndex }; } useProxy = true; if ( appConfig.http.addonProxyConfig && Object.keys(appConfig.http.addonProxyConfig).length > 0 ) { for (const [ruleHostname, ruleValue] of Object.entries( appConfig.http.addonProxyConfig )) { const ruleProxyIndexOrBool = String(ruleValue); if ( ['true', 'false'].includes(ruleProxyIndexOrBool) === false && isNaN(parseInt(ruleProxyIndexOrBool)) ) { logger.error( { hostname: ruleHostname, value: ruleProxyIndexOrBool }, 'invalid proxy config value' ); continue; } if (ruleHostname === '*') { useProxy = !(ruleProxyIndexOrBool === 'false'); proxyIndex = Number.isInteger(parseInt(ruleProxyIndexOrBool)) ? parseInt(ruleProxyIndexOrBool) : ruleProxyIndexOrBool === 'true' ? 0 : -1; } else if (ruleHostname.startsWith('*')) { if (hostname.endsWith(ruleHostname.slice(1))) { useProxy = !(ruleProxyIndexOrBool === 'false'); proxyIndex = Number.isInteger(parseInt(ruleProxyIndexOrBool)) ? parseInt(ruleProxyIndexOrBool) : ruleProxyIndexOrBool === 'true' ? 0 : -1; } } if (hostname === ruleHostname) { useProxy = !(ruleProxyIndexOrBool === 'false'); proxyIndex = Number.isInteger(parseInt(ruleProxyIndexOrBool)) ? parseInt(ruleProxyIndexOrBool) : ruleProxyIndexOrBool === 'true' ? 0 : -1; } } } else { proxyIndex = 0; } if (useProxy && appConfig.http.addonProxy[proxyIndex] === undefined) { logger.error({ proxyIndex }, 'proxy index out of range'); return { useProxy: false, proxyIndex: -1 }; } return { useProxy, proxyIndex }; } export function domainHasUserAgent(url: URL) { let userAgent: string | undefined; let hostname = url.hostname; if ( !appConfig.http.hostnameUserAgentOverrides || Object.keys(appConfig.http.hostnameUserAgentOverrides).length === 0 ) { return undefined; } const mappings = Object.entries(appConfig.http.hostnameUserAgentOverrides); for (const [ruleHostname, ruleUserAgent] of mappings) { if (ruleHostname === '*') { userAgent = ruleUserAgent; } else if (ruleHostname.startsWith('*')) { if (hostname.endsWith(ruleHostname.slice(1))) { userAgent = ruleUserAgent; } } else if (hostname === ruleHostname) { userAgent = ruleUserAgent; } } return userAgent; }