clash-linux commited on
Commit
c99d2c6
·
verified ·
1 Parent(s): 25fc068

Upload 17 files

Browse files
Files changed (5) hide show
  1. Dockerfile +17 -1
  2. entrypoint.sh +55 -0
  3. src/ProxyPool.js +440 -82
  4. src/ProxyServer.js +5 -18
  5. src/lightweight-client.js +21 -8
Dockerfile CHANGED
@@ -16,6 +16,19 @@ RUN npm install --production
16
  # Copy the rest of the application source code to the working directory
17
  COPY . .
18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  # Grant execute permission to the proxy server binary for the root user
20
  RUN chmod +x /app/src/proxy/chrome_proxy_server_linux_amd64
21
 
@@ -28,5 +41,8 @@ USER node
28
  # Make port 7860 available to the world outside this container
29
  EXPOSE 7860
30
 
31
- # Define the command to run the app
 
 
 
32
  CMD ["npm", "start"]
 
16
  # Copy the rest of the application source code to the working directory
17
  COPY . .
18
 
19
+ # Switch to root to install packages and set up the environment
20
+ USER root
21
+
22
+ # Install proxychains-ng for transparent proxying
23
+ RUN apt-get update && apt-get install -y --no-install-recommends proxychains-ng && rm -rf /var/lib/apt/lists/*
24
+
25
+ # Copy the entrypoint script and make it executable
26
+ COPY entrypoint.sh /app/entrypoint.sh
27
+ RUN chmod +x /app/entrypoint.sh
28
+
29
+ # Create a directory for proxychains configuration for the 'node' user
30
+ RUN mkdir -p /home/node/.proxychains && chown -R node:node /home/node/.proxychains
31
+
32
  # Grant execute permission to the proxy server binary for the root user
33
  RUN chmod +x /app/src/proxy/chrome_proxy_server_linux_amd64
34
 
 
41
  # Make port 7860 available to the world outside this container
42
  EXPOSE 7860
43
 
44
+ # Use the entrypoint script to start the application
45
+ ENTRYPOINT ["/app/entrypoint.sh"]
46
+
47
+ # Define the default command to run the app (will be passed to the entrypoint)
48
  CMD ["npm", "start"]
entrypoint.sh ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/sh
2
+ set -e
3
+
4
+ # This script configures and runs the application with proxychains if PROXY_URL is set.
5
+ # It is designed to be the ENTRYPOINT of a Docker container.
6
+
7
+ if [ -n "$PROXY_URL" ]; then
8
+ echo "PROXY_URL is set. Configuring proxychains..."
9
+
10
+ # Extract details from PROXY_URL
11
+ # Format: http://user:pass@host:port
12
+ PROTO=$(echo "$PROXY_URL" | grep :// | sed -e's,^\(.*://\).*,\1,g' | sed -e 's,://,,g')
13
+ URL_NO_PROTO=$(echo "$PROXY_URL" | sed -e 's,^.*://,,g')
14
+ USER_PASS=$(echo "$URL_NO_PROTO" | grep -o '.*@' | sed 's/@//')
15
+ HOST_PORT=$(echo "$URL_NO_PROTO" | sed -e "s,$USER_PASS@,,g")
16
+ HOST=$(echo "$HOST_PORT" | cut -d: -f1)
17
+ PORT=$(echo "$HOST_PORT" | cut -d: -f2)
18
+
19
+ # Default to http if protocol is not supported by proxychains
20
+ if [ "$PROTO" != "http" ] && [ "$PROTO" != "socks4" ] && [ "$PROTO" != "socks5" ]; then
21
+ echo "Warning: Unsupported proxy type '$PROTO'. Defaulting to 'http'."
22
+ PROTO="http"
23
+ fi
24
+
25
+ # Create proxychains config in the home directory of the 'node' user
26
+ CONFIG_DIR="/home/node/.proxychains"
27
+ mkdir -p "$CONFIG_DIR"
28
+ CONFIG_FILE="$CONFIG_DIR/proxychains.conf"
29
+
30
+ # Write base config
31
+ cat > "$CONFIG_FILE" <<EOF
32
+ strict_chain
33
+ proxy_dns
34
+ [ProxyList]
35
+ EOF
36
+
37
+ # Add proxy server entry
38
+ if [ -n "$USER_PASS" ]; then
39
+ USER=$(echo "$USER_PASS" | cut -d: -f1)
40
+ PASS=$(echo "$USER_PASS" | cut -d: -f2)
41
+ echo "$PROTO $HOST $PORT $USER $PASS" >> "$CONFIG_FILE"
42
+ echo "Configured proxy: $PROTO://$USER_PASS@$HOST:$PORT"
43
+ else
44
+ echo "$PROTO $HOST $PORT" >> "$CONFIG_FILE"
45
+ echo "Configured proxy: $PROTO://$HOST:$PORT"
46
+ fi
47
+
48
+ echo "Starting application with proxychains..."
49
+ # Execute the command passed to the script, wrapped in proxychains
50
+ exec proxychains4 "$@"
51
+ else
52
+ echo "PROXY_URL is not set. Starting application without proxy."
53
+ # Execute the command as is
54
+ exec "$@"
55
+ fi
src/ProxyPool.js CHANGED
@@ -1,166 +1,524 @@
1
  import axios from 'axios';
2
- import dotenv from 'dotenv';
3
- import { fileURLToPath } from 'url';
4
- import { dirname, join } from 'path';
5
-
6
- // 获取当前文件的目录路径
7
- const __filename = fileURLToPath(import.meta.url);
8
- const __dirname = dirname(__filename);
9
-
10
- // 加载环境变量
11
- dotenv.config({ path: join(dirname(__dirname), '.env') });
12
 
13
  /**
14
- * 代理管理类,用于提供一个静态的HTTP代理
15
  */
16
  class ProxyPool {
17
  /**
18
- * 创建代理管理器实例
19
- * @param {Object} options - 配置选项 (大部分将被忽略)
20
- * @param {string} options.targetUrl - 测试代理用的目标网站URL
21
- * @param {number} options.requestTimeout - 请求目标网站超时时间(毫秒)
 
 
 
 
 
 
 
 
 
 
 
22
  */
23
  constructor(options = {}) {
24
- this.staticProxy = process.env.STATIC_PROXY;
25
- this.targetUrl = options.targetUrl || 'https://www.notion.so';
 
 
26
  this.requestTimeout = options.requestTimeout || 10000;
 
 
 
 
 
 
 
 
 
27
 
28
- this.proxy = null; // 将存储解析后的代理对象
 
 
29
  this.isInitialized = false;
30
-
 
 
 
31
  // 绑定方法
32
  this.getProxy = this.getProxy.bind(this);
33
  this.removeProxy = this.removeProxy.bind(this);
 
34
  }
35
 
36
  /**
37
- * 初始化代理
38
  * @returns {Promise<void>}
39
  */
40
  async initialize() {
41
  if (this.isInitialized) return;
42
-
43
- if (!this.staticProxy) {
44
- console.error('错误: 环境变量 STATIC_PROXY 未设置。无法初始化代理。');
45
- this.isInitialized = true;
46
- return;
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
 
49
- console.log(`初始化静态代理: ${this.staticProxy}`);
 
 
 
50
 
51
  try {
52
- const url = new URL(this.staticProxy);
53
- const protocol = url.protocol.slice(0, -1);
54
- const proxyUrlForTest = `${url.hostname}:${url.port}`;
55
 
56
- const isValid = await this.testProxy(proxyUrlForTest, protocol);
57
-
58
- if (isValid) {
59
- this.proxy = {
60
- ip: url.hostname,
61
- port: url.port,
62
- protocol: protocol,
63
- full: this.staticProxy,
64
- addedAt: new Date().toISOString()
65
- };
66
- console.log(`静态代理 ${this.staticProxy} 已验证并准备就绪。`);
67
- } else {
68
- console.warn(`警告: 静态代理 ${this.staticProxy} 测试失败,当前不可用。后续将继续尝试使用。`);
69
- // 即使测试失败,也设置代理对象,便 getProxy 可以返回它
70
- // 调用方需要处理请求失败的情况
71
- this.proxy = {
72
- ip: url.hostname,
73
- port: url.port,
74
- protocol: protocol,
75
- full: this.staticProxy,
76
- };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  }
78
  } catch (error) {
79
- console.error(`无效的静态代理URL '${this.staticProxy}': ${error.message}`);
80
- // 阻止程序继续运行,因为代理配置是关键
81
  } finally {
82
- this.isInitialized = true;
 
 
 
 
 
 
83
  }
84
  }
85
 
86
  /**
87
- * 停止代理服务 (空方法,为保持接口一致性)
 
88
  */
89
- stop() {
90
- console.log('代理服务已停止');
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  }
92
 
93
  /**
94
  * 测试代理是否可用
95
- * @param {string} proxyUrl - 代理URL (ip:port)
96
- * @param {string} protocol - 代理协议
97
  * @returns {Promise<boolean>} - 代理是否可用
98
  */
99
- async testProxy(proxyUrl, protocol) {
100
  try {
101
- const [host, port] = proxyUrl.split(':');
102
  const proxyConfig = {
103
- host,
104
- port: parseInt(port),
105
- protocol,
106
  };
107
 
 
108
  const response = await axios.get(this.targetUrl, {
109
  proxy: proxyConfig,
110
  headers: {
111
  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
 
 
 
 
112
  },
113
  timeout: this.requestTimeout,
114
- validateStatus: (status) => status === 200,
 
 
115
  });
116
 
117
- console.log(`代理 ${proxyUrl} 测试成功。`);
118
- return true;
 
 
 
 
 
 
 
 
 
 
 
 
 
119
  } catch (error) {
120
- console.log(`代理 ${proxyUrl} 测试出错: ${error.message}`);
121
  return false;
122
  }
123
  }
124
 
125
  /**
126
- * 获取静态代理
127
- * @returns {Object|null} - 代理对象,如果没有配置则返回null
128
  */
129
  getProxy() {
130
- if (!this.proxy) {
131
- if (this.staticProxy) {
132
- console.warn('静态代理正在初始化或初始化失败,请稍后重试。');
133
- } else {
134
- console.error('未配置静态代理 (STATIC_PROXY)。');
135
- }
136
  }
137
- return this.proxy;
 
 
 
 
 
138
  }
139
 
140
  /**
141
- * 移除代理 (现在只记录日志,不实际移除)
142
  * @param {string} ip - 代理IP
143
  * @param {string|number} port - 代理端口
 
144
  */
145
  removeProxy(ip, port) {
146
- console.warn(`收到移除代理 ${ip}:${port} 的请求。由于使用的是静态代理,该请求被忽略。请检查调用方逻辑。`);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
  }
148
 
149
  /**
150
- * 获取所有可用代理 (只会有一个)
151
  * @returns {Array<Object>} - 代理对象数组
152
  */
153
  getAllProxies() {
154
- return this.proxy ? [this.proxy] : [];
155
  }
156
 
157
  /**
158
- * 获取可用代理数量 (0或1)
159
  * @returns {number} - 代理数量
160
  */
161
  getCount() {
162
- return this.proxy ? 1 : 0;
163
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
  }
165
 
166
  // 导出 ProxyPool 类和实例
 
1
  import axios from 'axios';
 
 
 
 
 
 
 
 
 
 
2
 
3
  /**
4
+ * 代理类,用于管理和提供HTTP代理
5
  */
6
  class ProxyPool {
7
  /**
8
+ * 创建代理实例
9
+ * @param {Object} options - 配置选项
10
+ * @param {number} options.targetCount - 目标代理数量,默认20
11
+ * @param {number} options.batchSize - 每次获取代理数量,默认20
12
+ * @param {number} options.testTimeout - 测试代理超时时间(毫秒),默认5000
13
+ * @param {number} options.requestTimeout - 请求目标网站超时时间(毫秒),默认10000
14
+ * @param {string} options.targetUrl - 目标网站URL,默认'https://www.notion.so'
15
+ * @param {number} options.concurrentRequests - 并发请求数量,默认10
16
+ * @param {number} options.minThreshold - 可用代理数量低于此阈值时自动补充,默认5
17
+ * @param {number} options.checkInterval - 检查代理池状态的时间间隔(毫秒),默认30000
18
+ * @param {string} options.proxyProtocol - 代理协议,默认'http'
19
+ * @param {number} options.maxRefillAttempts - 最大补充尝试次数,默认20
20
+ * @param {number} options.retryDelay - 重试延迟(毫秒),默认1000
21
+ * @param {boolean} options.useCache - 是否使用缓存,默认true
22
+ * @param {number} options.cacheExpiry - 缓存过期时间(毫秒),默认3600000 (1小时)
23
  */
24
  constructor(options = {}) {
25
+ // 配置参数
26
+ this.targetCount = options.targetCount || 20;
27
+ this.batchSize = options.batchSize || 20;
28
+ this.testTimeout = options.testTimeout || 5000;
29
  this.requestTimeout = options.requestTimeout || 10000;
30
+ this.targetUrl = options.targetUrl || 'https://www.notion.so';
31
+ this.concurrentRequests = options.concurrentRequests || 10;
32
+ this.minThreshold = options.minThreshold || 5;
33
+ this.checkInterval = options.checkInterval || 30000; // 默认30秒检查一次
34
+ this.proxyProtocol = options.proxyProtocol || 'http';
35
+ this.maxRefillAttempts = options.maxRefillAttempts || 20; // 减少最大尝试次数
36
+ this.retryDelay = options.retryDelay || 1000; // 减少重试延迟
37
+ this.useCache = options.useCache !== undefined ? options.useCache : true;
38
+ this.cacheExpiry = options.cacheExpiry || 3600000; // 默认1小时
39
 
40
+ // 内部状态
41
+ this.availableProxies = [];
42
+ this.currentIndex = 0;
43
  this.isInitialized = false;
44
+ this.isRefilling = false;
45
+ this.checkTimer = null;
46
+ this.proxyCache = new Map(); // 缓存验证过的代理
47
+
48
  // 绑定方法
49
  this.getProxy = this.getProxy.bind(this);
50
  this.removeProxy = this.removeProxy.bind(this);
51
+ this.checkAndRefill = this.checkAndRefill.bind(this);
52
  }
53
 
54
  /**
55
+ * 初始化代理
56
  * @returns {Promise<void>}
57
  */
58
  async initialize() {
59
  if (this.isInitialized) return;
60
+
61
+ console.log(`初始化代理池,目标数量: ${this.targetCount}`);
62
+ await this.refillProxies();
63
+
64
+ // 设置定时检查
65
+ this.checkTimer = setInterval(this.checkAndRefill, this.checkInterval);
66
+
67
+ this.isInitialized = true;
68
+ console.log(`代理池初始化完成,当前可用代理数量: ${this.availableProxies.length}`);
69
+ }
70
+
71
+ /**
72
+ * 停止代理池服务
73
+ */
74
+ stop() {
75
+ if (this.checkTimer) {
76
+ clearInterval(this.checkTimer);
77
+ this.checkTimer = null;
78
  }
79
+ console.log('代理池服务已停止');
80
+ }
81
+
82
+ /**
83
+ * 检查并补充代理
84
+ */
85
+ async checkAndRefill() {
86
+ if (this.availableProxies.length <= this.minThreshold && !this.isRefilling) {
87
+ console.log(`可用代理数量(${this.availableProxies.length})低于阈值(${this.minThreshold}),开始补充代理`);
88
+ await this.refillProxies();
89
+ }
90
+ }
91
+
92
+ /**
93
+ * 补充代理到目标数量
94
+ * @returns {Promise<void>}
95
+ */
96
+ async refillProxies() {
97
+ if (this.isRefilling) return;
98
 
99
+ this.isRefilling = true;
100
+ console.log(`开始补充代理,当前数量: ${this.availableProxies.length},目标���量: ${this.targetCount}`);
101
+
102
+ let attempts = 0;
103
 
104
  try {
105
+ // 计算需要补充的代理数量
106
+ const neededProxies = this.targetCount - this.availableProxies.length;
 
107
 
108
+ // 优先检查缓存中的代理
109
+ if (this.useCache && this.proxyCache.size > 0) {
110
+ await this.tryUsingCachedProxies(neededProxies);
111
+ }
112
+
113
+ // 如果缓存中的代理不足,继续获取新代理
114
+ while (this.availableProxies.length < this.targetCount && attempts < this.maxRefillAttempts) {
115
+ attempts++;
116
+
117
+ console.log(`补充尝试 #${attempts},当前可用代理: ${this.availableProxies.length}/${this.targetCount}`);
118
+
119
+ // 计算本次需要获取的批次大小
120
+ const remainingNeeded = this.targetCount - this.availableProxies.length;
121
+ const batchSizeNeeded = Math.max(this.batchSize, remainingNeeded * 2); // 获取更多代理以提高成功率
122
+
123
+ // 获取代理
124
+ const proxies = await this.getProxiesFromProvider(batchSizeNeeded);
125
+
126
+ if (proxies.length === 0) {
127
+ console.log(`没有获取到代理,等待${this.retryDelay/1000}秒后重试...`);
128
+ await new Promise(resolve => setTimeout(resolve, this.retryDelay));
129
+ continue;
130
+ }
131
+
132
+ // 过滤掉已有的代理
133
+ const newProxies = this.filterExistingProxies(proxies);
134
+
135
+ if (newProxies.length === 0) {
136
+ console.log('所有获取的代理都已存在,继续获取新代理...');
137
+ continue;
138
+ }
139
+
140
+ // 测试代理
141
+ const results = await this.testProxiesConcurrently(newProxies);
142
+
143
+ // 添加可用代理
144
+ this.addValidProxies(results);
145
+
146
+ // 如果已经获取到足够的代理,提前结束
147
+ if (this.availableProxies.length >= this.targetCount) {
148
+ break;
149
+ }
150
+
151
+ // 如果还没补充到足够的代理,等待一段时间再继续
152
+ if (this.availableProxies.length < this.targetCount) {
153
+ await new Promise(resolve => setTimeout(resolve, this.retryDelay));
154
+ }
155
  }
156
  } catch (error) {
157
+ console.error('补充代理过程中出错:', error);
 
158
  } finally {
159
+ this.isRefilling = false;
160
+
161
+ if (this.availableProxies.length >= this.targetCount) {
162
+ console.log(`代理补充完成,当前可用代理: ${this.availableProxies.length}/${this.targetCount}`);
163
+ } else {
164
+ console.log(`已达到最大尝试次数 ${this.maxRefillAttempts},当前可用代理: ${this.availableProxies.length}/${this.targetCount}`);
165
+ }
166
  }
167
  }
168
 
169
  /**
170
+ * 尝试使用缓存中的代理
171
+ * @param {number} neededProxies - 需要的代理数量
172
  */
173
+ async tryUsingCachedProxies(neededProxies) {
174
+ const now = Date.now();
175
+ const cachedProxies = [];
176
+
177
+ // 筛选未过期的缓存代理
178
+ for (const [proxyKey, data] of this.proxyCache.entries()) {
179
+ if (now - data.timestamp < this.cacheExpiry && data.valid) {
180
+ cachedProxies.push(proxyKey);
181
+
182
+ if (cachedProxies.length >= neededProxies) {
183
+ break;
184
+ }
185
+ }
186
+ }
187
+
188
+ if (cachedProxies.length > 0) {
189
+ console.log(`从缓存中找到 ${cachedProxies.length} 个可能可用的代理`);
190
+
191
+ // 验证缓存的代理是否仍然可用
192
+ const results = await this.testProxiesConcurrently(cachedProxies);
193
+ this.addValidProxies(results);
194
+ }
195
+ }
196
+
197
+ /**
198
+ * 过滤掉已存在的代理
199
+ * @param {Array<string>} proxies - 代理列表
200
+ * @returns {Array<string>} - 新代理列表
201
+ */
202
+ filterExistingProxies(proxies) {
203
+ return proxies.filter(proxy => {
204
+ const [ip, port] = proxy.split(':');
205
+ return !this.availableProxies.some(p => p.ip === ip && p.port === port);
206
+ });
207
+ }
208
+
209
+ /**
210
+ * 添加有效的代理到代理池
211
+ * @param {Array<{proxy: string, result: boolean}>} results - 测试结果
212
+ */
213
+ addValidProxies(results) {
214
+ for (const { proxy, result } of results) {
215
+ if (result) {
216
+ const [ip, port] = proxy.split(':');
217
+
218
+ // 检查是否已存在
219
+ if (!this.availableProxies.some(p => p.ip === ip && p.port === port)) {
220
+ const proxyObj = {
221
+ ip,
222
+ port,
223
+ protocol: this.proxyProtocol,
224
+ full: `${this.proxyProtocol}://${proxy}`,
225
+ addedAt: new Date().toISOString()
226
+ };
227
+
228
+ this.availableProxies.push(proxyObj);
229
+
230
+ // 添加到缓存
231
+ if (this.useCache) {
232
+ this.proxyCache.set(proxy, {
233
+ valid: true,
234
+ timestamp: Date.now()
235
+ });
236
+ }
237
+
238
+ console.log(`成功添加代理: ${proxyObj.full},当前可用代理: ${this.availableProxies.length}/${this.targetCount}`);
239
+
240
+ if (this.availableProxies.length >= this.targetCount) {
241
+ break;
242
+ }
243
+ }
244
+ } else if (this.useCache) {
245
+ // 记录无效代理到缓存
246
+ this.proxyCache.set(proxy, {
247
+ valid: false,
248
+ timestamp: Date.now()
249
+ });
250
+ }
251
+ }
252
+ }
253
+
254
+ /**
255
+ * 从代理服务获取代理URL
256
+ * @param {number} count - 请求的代理数量
257
+ * @returns {Promise<Array<string>>} - 代理URL列表
258
+ */
259
+ async getProxiesFromProvider(count = null) {
260
+ try {
261
+ const requestCount = count || this.batchSize;
262
+ const url = `https://proxy.scdn.io/api/get_proxy.php?protocol=${this.proxyProtocol}&count=${requestCount}`;
263
+ console.log(`正在获取代理,URL: ${url}`);
264
+
265
+ const response = await axios.get(url, {
266
+ timeout: 10000,
267
+ validateStatus: status => true
268
+ });
269
+
270
+ if (response.data && response.data.code === 200) {
271
+ console.log(`成功获取 ${response.data.data.count} 个代理`);
272
+ return response.data.data.proxies;
273
+ } else {
274
+ console.error('获取代理失败:', response.data ? response.data.message : '未知错误');
275
+ return [];
276
+ }
277
+ } catch (error) {
278
+ console.error('获取代理出错:', error.message);
279
+ return [];
280
+ }
281
+ }
282
+
283
+ /**
284
+ * 并发测试多个代理
285
+ * @param {Array<string>} proxies - 代理列表
286
+ * @returns {Promise<Array<{proxy: string, result: boolean}>>} - 测试结果
287
+ */
288
+ async testProxiesConcurrently(proxies) {
289
+ const results = [];
290
+ const remainingNeeded = this.targetCount - this.availableProxies.length;
291
+
292
+ // 增加并发数以加快处理速度
293
+ const concurrentRequests = Math.min(this.concurrentRequests * 2, 20);
294
+
295
+ // 分批处理代理
296
+ for (let i = 0; i < proxies.length; i += concurrentRequests) {
297
+ const batch = proxies.slice(i, i + concurrentRequests);
298
+ const promises = batch.map(proxy => {
299
+ // 检查缓存中是否有近期验证过的结果
300
+ if (this.useCache && this.proxyCache.has(proxy)) {
301
+ const cachedResult = this.proxyCache.get(proxy);
302
+ const isFresh = (Date.now() - cachedResult.timestamp) < this.cacheExpiry;
303
+
304
+ if (isFresh) {
305
+ // 使用缓存结果,避免重复测试
306
+ return Promise.resolve({ proxy, result: cachedResult.valid });
307
+ }
308
+ }
309
+
310
+ return this.testProxy(proxy)
311
+ .then(result => ({ proxy, result }))
312
+ .catch(() => ({ proxy, result: false }));
313
+ });
314
+
315
+ const batchResults = await Promise.all(promises);
316
+ results.push(...batchResults);
317
+
318
+ // 如果已经找到足够的代理,提前结束测试
319
+ const successCount = results.filter(item => item.result).length;
320
+ if (successCount >= remainingNeeded) {
321
+ break;
322
+ }
323
+ }
324
+
325
+ return results;
326
  }
327
 
328
  /**
329
  * 测试代理是否可用
330
+ * @param {string} proxyUrl - 代理URL
 
331
  * @returns {Promise<boolean>} - 代理是否可用
332
  */
333
+ async testProxy(proxyUrl) {
334
  try {
335
+ // 创建代理配置
336
  const proxyConfig = {
337
+ host: proxyUrl.split(':')[0],
338
+ port: parseInt(proxyUrl.split(':')[1]),
339
+ protocol: this.proxyProtocol
340
  };
341
 
342
+ // 发送请求到目标网站
343
  const response = await axios.get(this.targetUrl, {
344
  proxy: proxyConfig,
345
  headers: {
346
  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
347
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
348
+ 'Accept-Language': 'en-US,en;q=0.5',
349
+ 'Connection': 'keep-alive',
350
+ 'Upgrade-Insecure-Requests': '1'
351
  },
352
  timeout: this.requestTimeout,
353
+ validateStatus: status => true,
354
+ maxRedirects: 10,
355
+ followRedirect: true
356
  });
357
 
358
+ // 检查响应是否包含目标网站特有的内容
359
+ const isTargetContent = response.data &&
360
+ (typeof response.data === 'string') &&
361
+ (response.data.includes('notion') ||
362
+ response.data.includes('Notion'));
363
+
364
+ const isValid = response.status === 200 && isTargetContent;
365
+
366
+ if (isValid) {
367
+ console.log(`代理 ${proxyUrl} 请求目标网站成功,状态码: ${response.status}`);
368
+ } else {
369
+ console.log(`代理 ${proxyUrl} 请求目标网站失败,状态码: ${response.status}`);
370
+ }
371
+
372
+ return isValid;
373
  } catch (error) {
374
+ console.log(`代理 ${proxyUrl} 请求出错: ${error.message}`);
375
  return false;
376
  }
377
  }
378
 
379
  /**
380
+ * 获取一个可用代理
381
+ * @returns {Object|null} - 代理对象,如果没有可用代理则返回null
382
  */
383
  getProxy() {
384
+ if (this.availableProxies.length === 0) {
385
+ console.log('没有可用代理');
386
+ return null;
 
 
 
387
  }
388
+
389
+ // 轮询方式获取代理
390
+ const proxy = this.availableProxies[this.currentIndex];
391
+ this.currentIndex = (this.currentIndex + 1) % this.availableProxies.length;
392
+
393
+ return proxy;
394
  }
395
 
396
  /**
397
+ * 移除指定代理
398
  * @param {string} ip - 代理IP
399
  * @param {string|number} port - 代理端口
400
+ * @returns {boolean} - 是否成功移除
401
  */
402
  removeProxy(ip, port) {
403
+ const portStr = port.toString();
404
+ const initialLength = this.availableProxies.length;
405
+
406
+ // 找到要移除的代理
407
+ const proxyToRemove = this.availableProxies.find(
408
+ proxy => proxy.ip === ip && proxy.port === portStr
409
+ );
410
+
411
+ if (proxyToRemove) {
412
+ // 更新缓存,标记为无效
413
+ if (this.useCache) {
414
+ const proxyKey = `${ip}:${portStr}`;
415
+ this.proxyCache.set(proxyKey, { valid: false, timestamp: Date.now() });
416
+ }
417
+ }
418
+
419
+ this.availableProxies = this.availableProxies.filter(
420
+ proxy => !(proxy.ip === ip && proxy.port === portStr)
421
+ );
422
+
423
+ // 重置当前索引,确保不会越界
424
+ if (this.currentIndex >= this.availableProxies.length && this.availableProxies.length > 0) {
425
+ this.currentIndex = 0;
426
+ }
427
+
428
+ const removed = initialLength > this.availableProxies.length;
429
+
430
+ if (removed) {
431
+ console.log(`已移除代理 ${ip}:${port},当前可用代理: ${this.availableProxies.length}`);
432
+ } else {
433
+ console.log(`未找到要移除的代理 ${ip}:${port}`);
434
+ }
435
+
436
+ // 如果移除后代理数量低于阈值,触发补充
437
+ this.checkAndRefill();
438
+
439
+ return removed;
440
  }
441
 
442
  /**
443
+ * 获取所有可用代理
444
  * @returns {Array<Object>} - 代理对象数组
445
  */
446
  getAllProxies() {
447
+ return [...this.availableProxies];
448
  }
449
 
450
  /**
451
+ * 获取可用代理数量
452
  * @returns {number} - 代理数量
453
  */
454
  getCount() {
455
+ return this.availableProxies.length;
456
  }
457
+
458
+ /**
459
+ * 清理过期的缓存条目
460
+ */
461
+ cleanupCache() {
462
+ if (!this.useCache) return;
463
+
464
+ const now = Date.now();
465
+ let cleanupCount = 0;
466
+
467
+ for (const [key, data] of this.proxyCache.entries()) {
468
+ if (now - data.timestamp > this.cacheExpiry) {
469
+ this.proxyCache.delete(key);
470
+ cleanupCount++;
471
+ }
472
+ }
473
+
474
+ if (cleanupCount > 0) {
475
+ console.log(`清理了 ${cleanupCount} 个过期的缓存代理`);
476
+ }
477
+ }
478
+ }
479
+
480
+ // 使用示例
481
+ async function example() {
482
+ // 创建代理池实例
483
+ const proxyPool = new ProxyPool({
484
+ targetCount: 10, // 目标保持10个代理
485
+ minThreshold: 3, // 当可用代理少于3个时,自动补充
486
+ checkInterval: 60000, // 每60秒检查一次
487
+ targetUrl: 'https://www.notion.so',
488
+ concurrentRequests: 15, // 增加并发请求数
489
+ useCache: true, // 启用缓存
490
+ maxRefillAttempts: 15, // 减少最大尝试次数
491
+ retryDelay: 1000 // 减少重试延迟
492
+ });
493
+
494
+ // 初始化代理池
495
+ await proxyPool.initialize();
496
+
497
+ // 获取一个代理
498
+ const proxy = proxyPool.getProxy();
499
+ console.log('获取到代理:', proxy);
500
+
501
+ // 模拟使用一段时间后,移除一个代理
502
+ setTimeout(() => {
503
+ if (proxy) {
504
+ proxyPool.removeProxy(proxy.ip, proxy.port);
505
+ }
506
+
507
+ // 获取所有代理
508
+ const allProxies = proxyPool.getAllProxies();
509
+ console.log(`当前所有代理(${allProxies.length}):`, allProxies);
510
+
511
+ // 使用完毕后停止服务
512
+ setTimeout(() => {
513
+ proxyPool.stop();
514
+ console.log('代理池示例运行完毕');
515
+ }, 5000);
516
+ }, 5000);
517
+ }
518
+
519
+ // 如果直接运行此文件,则执行示例
520
+ if (typeof require !== 'undefined' && require.main === module) {
521
+ example().catch(err => console.error('示例运行出错:', err));
522
  }
523
 
524
  // 导出 ProxyPool 类和实例
src/ProxyServer.js CHANGED
@@ -106,36 +106,23 @@ class ProxyServer {
106
  // 创建日志文件
107
  this.logStream = fs.createWriteStream(this.logPath, { flags: 'a' });
108
 
109
- const spawnArgs = [
 
 
110
  '--port', this.port.toString(),
111
  '--token', this.proxyAuthToken
112
- ];
113
-
114
- const staticProxy = process.env.STATIC_PROXY;
115
- if (staticProxy) {
116
- spawnArgs.push(`--proxy-server=${staticProxy}`);
117
- logger.info(`为 chrome_proxy_server 配置上游代理: ${staticProxy}`);
118
- }
119
-
120
- // 启动代理服务器进程
121
- this.proxyProcess = spawn(proxyServerPath, spawnArgs, {
122
- stdio: ['ignore', 'pipe', 'pipe'],
123
  detached: false
124
  });
125
 
126
  // 将进程的输出重定向到日志文件
127
  if (this.proxyProcess.stdout) {
128
  this.proxyProcess.stdout.pipe(this.logStream);
129
- this.proxyProcess.stdout.on('data', (data) => {
130
- logger.info(`[ProxyExecutable STDOUT] ${data.toString().trim()}`);
131
- });
132
  }
133
 
134
  if (this.proxyProcess.stderr) {
135
  this.proxyProcess.stderr.pipe(this.logStream);
136
- this.proxyProcess.stderr.on('data', (data) => {
137
- logger.error(`[ProxyExecutable STDERR] ${data.toString().trim()}`);
138
- });
139
  }
140
 
141
  // 设置进程事件处理
 
106
  // 创建日志文件
107
  this.logStream = fs.createWriteStream(this.logPath, { flags: 'a' });
108
 
109
+ // 修复 stdio 参数问题
110
+ // 启动代理服务器进程
111
+ this.proxyProcess = spawn(proxyServerPath, [
112
  '--port', this.port.toString(),
113
  '--token', this.proxyAuthToken
114
+ ], {
115
+ stdio: ['ignore', 'pipe', 'pipe'], // 使用pipe而不是直接传递流
 
 
 
 
 
 
 
 
 
116
  detached: false
117
  });
118
 
119
  // 将进程的输出重定向到日志文件
120
  if (this.proxyProcess.stdout) {
121
  this.proxyProcess.stdout.pipe(this.logStream);
 
 
 
122
  }
123
 
124
  if (this.proxyProcess.stderr) {
125
  this.proxyProcess.stderr.pipe(this.logStream);
 
 
 
126
  }
127
 
128
  // 设置进程事件处理
src/lightweight-client.js CHANGED
@@ -34,10 +34,13 @@ const logger = {
34
  const NOTION_API_URL = "https://www.notion.so/api/v3/runInferenceTranscript";
35
  // 这些变量将由cookieManager动态提供
36
  let currentCookieData = null;
 
37
  const ENABLE_PROXY_SERVER = process.env.ENABLE_PROXY_SERVER === 'true';
38
- const STATIC_PROXY = process.env.STATIC_PROXY;
39
  let proxy = null;
40
 
 
 
 
41
  // 标记是否成功初始化
42
  let INITIALIZED_SUCCESSFULLY = false;
43
 
@@ -338,17 +341,22 @@ async function fetchNotionResponse(chunkQueue, notionRequestBody, headers, notio
338
  };
339
 
340
  // 添加代理配置(如果有)
341
- if (!ENABLE_PROXY_SERVER && STATIC_PROXY) {
342
  proxy = proxyPool.getProxy();
343
- if (proxy && proxy.full) {
 
344
  const { HttpsProxyAgent } = await import('https-proxy-agent');
345
  fetchOptions.agent = new HttpsProxyAgent(proxy.full);
346
- logger.info(`使用静态代理: ${proxy.full}`);
347
- } else {
348
- logger.warning(`无法从 proxyPool 获取到静态代理,将不使用代理进行请求。`);
 
349
  }
 
 
 
 
350
  }
351
-
352
  let response = null;
353
  // 发送请求
354
  if (ENABLE_PROXY_SERVER){
@@ -481,7 +489,7 @@ async function fetchNotionResponse(chunkQueue, notionRequestBody, headers, notio
481
  // 如果没有收到任何响应,发送一个提示消息
482
  if (!responseReceived) {
483
  logger.warning(`未从Notion收到内容响应,请更换ip重试`);
484
- if (ENABLE_PROXY_SERVER) {
485
  proxyPool.removeProxy(proxy.ip, proxy.port);
486
  }
487
 
@@ -733,6 +741,11 @@ async function initialize() {
733
  logger.info(`当前使用的cookie对应的用户ID: ${currentCookieData.userId}`);
734
  logger.info(`当前使用的cookie对应的空间ID: ${currentCookieData.spaceId}`);
735
 
 
 
 
 
 
736
  INITIALIZED_SUCCESSFULLY = true;
737
  }
738
 
 
34
  const NOTION_API_URL = "https://www.notion.so/api/v3/runInferenceTranscript";
35
  // 这些变量将由cookieManager动态提供
36
  let currentCookieData = null;
37
+ const USE_NATIVE_PROXY_POOL = process.env.USE_NATIVE_PROXY_POOL === 'true';
38
  const ENABLE_PROXY_SERVER = process.env.ENABLE_PROXY_SERVER === 'true';
 
39
  let proxy = null;
40
 
41
+ // 代理配置
42
+ const PROXY_URL = process.env.PROXY_URL || "";
43
+
44
  // 标记是否成功初始化
45
  let INITIALIZED_SUCCESSFULLY = false;
46
 
 
341
  };
342
 
343
  // 添加代理配置(如果有)
344
+ if (USE_NATIVE_PROXY_POOL) {
345
  proxy = proxyPool.getProxy();
346
+ if (proxy !== null)
347
+ {
348
  const { HttpsProxyAgent } = await import('https-proxy-agent');
349
  fetchOptions.agent = new HttpsProxyAgent(proxy.full);
350
+ logger.info(`使用代理: ${proxy.full}`);
351
+ }
352
+ else{
353
+ logger.warning(`没有可用代理`);
354
  }
355
+ } else if(PROXY_URL) {
356
+ const { HttpsProxyAgent } = await import('https-proxy-agent');
357
+ fetchOptions.agent = new HttpsProxyAgent(PROXY_URL);
358
+ logger.info(`使用代理: ${PROXY_URL}`);
359
  }
 
360
  let response = null;
361
  // 发送请求
362
  if (ENABLE_PROXY_SERVER){
 
489
  // 如果没有收到任何响应,发送一个提示消息
490
  if (!responseReceived) {
491
  logger.warning(`未从Notion收到内容响应,请更换ip重试`);
492
+ if (USE_NATIVE_PROXY_POOL) {
493
  proxyPool.removeProxy(proxy.ip, proxy.port);
494
  }
495
 
 
741
  logger.info(`当前使用的cookie对应的用户ID: ${currentCookieData.userId}`);
742
  logger.info(`当前使用的cookie对应的空间ID: ${currentCookieData.spaceId}`);
743
 
744
+ if (process.env.USE_NATIVE_PROXY_POOL === 'true') {
745
+ logger.info(`正在初始化本地代理池...`);
746
+ await proxyPool.initialize();
747
+ }
748
+
749
  INITIALIZED_SUCCESSFULLY = true;
750
  }
751