File size: 4,542 Bytes
bf48b89 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | import type { HeaderGeneratorOptions } from 'header-generator';
import { useRegisterRequest } from 'node-network-devtools';
import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
import type { RequestInfo, RequestInit } from 'undici';
import undici, { Request } from 'undici';
import { config } from '@/config';
import { generatedHeaders as HEADER_LIST, generateHeaders } from '@/utils/header-generator';
import logger from '@/utils/logger';
import proxy from '@/utils/proxy';
const limiter = new RateLimiterMemory({
points: 10,
duration: 1,
execEvenly: true,
});
const limiterQueue = new RateLimiterQueue(limiter, {
maxQueueSize: 4800,
});
export const useCustomHeader = (headers: Headers) => {
process.env.NODE_ENV === 'dev' &&
useRegisterRequest((req) => {
for (const [key, value] of headers.entries()) {
req.requestHeaders[key] = value;
}
return req;
});
};
const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit & { headerGeneratorOptions?: Partial<HeaderGeneratorOptions> }) => {
const request = new Request(input, init);
const options: RequestInit = {};
logger.debug(`Outgoing request: ${request.method} ${request.url}`);
const generatedHeaders = generateHeaders(init?.headerGeneratorOptions);
// ua
if (!request.headers.has('user-agent')) {
request.headers.set('user-agent', config.ua);
}
for (const header of HEADER_LIST) {
if (!request.headers.has(header) && generatedHeaders[header]) {
request.headers.set(header, generatedHeaders[header]);
}
}
// referer
if (!request.headers.get('referer')) {
try {
const urlHandler = new URL(request.url);
request.headers.set('referer', urlHandler.origin);
} catch {
// ignore
}
}
let isRetry = false;
if (request.headers.get('x-prefer-proxy')) {
isRetry = true;
request.headers.delete('x-prefer-proxy');
}
config.enableRemoteDebugging && useCustomHeader(request.headers);
// proxy
if (!init?.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
const proxyRegex = new RegExp(proxy.proxyObj.url_regex);
let urlHandler;
try {
urlHandler = new URL(request.url);
} catch {
// ignore
}
if (proxyRegex.test(request.url) && request.url.startsWith('http') && !(urlHandler && urlHandler.host === proxy.proxyUrlHandler?.host)) {
const currentProxy = proxy.getCurrentProxy();
if (currentProxy) {
const dispatcher = proxy.getDispatcherForProxy(currentProxy);
if (dispatcher) {
options.dispatcher = dispatcher;
logger.debug(`Proxying request via ${currentProxy.uri}: ${request.url}`);
}
}
}
}
await limiterQueue.removeTokens(1);
const maxRetries = proxy.multiProxy?.allProxies.length || 1;
const attemptRequest = async (attempt: number): Promise<Response> => {
try {
return await undici.fetch(request, options);
} catch (error) {
if (options.dispatcher && proxy.multiProxy && attempt < maxRetries - 1) {
const currentProxy = proxy.getCurrentProxy();
if (currentProxy) {
logger.warn(`Request failed with proxy ${currentProxy.uri}, trying next proxy: ${error}`);
proxy.markProxyFailed(currentProxy.uri);
const nextProxy = proxy.getCurrentProxy();
if (nextProxy && nextProxy.uri !== currentProxy.uri) {
const nextDispatcher = proxy.getDispatcherForProxy(nextProxy);
if (nextDispatcher) {
options.dispatcher = nextDispatcher;
}
logger.debug(`Retrying request with proxy ${nextProxy.uri}: ${request.url}`);
return attemptRequest(attempt + 1);
} else {
logger.warn('No more proxies available, trying without proxy');
delete options.dispatcher;
return attemptRequest(attempt + 1);
}
}
}
throw error;
}
};
return attemptRequest(0);
};
export default wrappedFetch;
|