Upload 15 files

.github/workflows/docker-publish.yml

@@ -0,0 +1,32 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build_and_push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ghcr.io/${{ github.repository }}:latest 

Dockerfile

@@ -1,55 +1,28 @@
 # Use an official Node.js runtime as a parent image
 FROM node:18-slim
 
-# Switch to root user to install packages and configure system
-USER root
-
 # Set the working directory in the container
 WORKDIR /app
 
-# Install dependencies needed for traffic redirection and privilege drop
-# And clean up apt-get lists to keep the image slim
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    iptables \
-    redsocks \
-    gosu \
-    && rm -rf /var/lib/apt/lists/*
-
 # Copy package.json and package-lock.json to the working directory
 COPY package*.json ./
 
 # Install app dependencies
-# Using --production flag to install only production dependencies
-# Playwright needs special handling to download browsers
-# We need to install dependencies first to make sure Playwright downloads its browsers.
 RUN npm install --production
 
 # Copy the rest of the application source code to the working directory
 COPY . .
 
-# Copy the configuration for redsocks
-COPY redsocks.conf /etc/redsocks.conf
-
-# Copy the entrypoint script
-COPY entrypoint.sh /app/entrypoint.sh
+# Grant execute permission to the proxy server binary for the root user
+# We keep this for now in case other parts of the app still reference it.
+# We will remove it in a later step.
+RUN chmod +x /app/src/proxy/chrome_proxy_server_linux_amd64
 
-# Grant execute permissions to scripts and the proxy binary
-RUN chmod +x /app/entrypoint.sh \
-    && chmod +x /app/src/proxy/chrome_proxy_server_linux_amd64
-
-# Change ownership of the app directory to the node user
-RUN chown -R node:node /app
-
-# Switch to a non-root user for security
+# Switch to a non-root user for security. This is a good practice.
 USER node
 
 # Make port 7860 available to the world outside this container
 EXPOSE 7860
 
-# Set the entrypoint to our script.
-# This script will run as root to set up iptables.
-ENTRYPOINT ["/app/entrypoint.sh"]
-
-# Define the default command to run the app.
-# This will be executed by the entrypoint script using 'gosu' to drop to the 'node' user.
+# Define the command to run the app
 CMD ["npm", "start"] 

package-lock.json

@@ -13,6 +13,7 @@
         "chalk": "^4.1.2",
         "dotenv": "^16.3.1",
         "express": "^4.18.2",
+        "got-scraping": "^4.1.1",
         "https-proxy-agent": "^7.0.2",
         "jsdom": "^22.1.0",
         "node-fetch": "^3.3.2",
@@ -26,6 +27,33 @@
         "nodemon": "^3.0.2"
       }
     },
+    "node_modules/@sec-ant/readable-stream": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz",
+      "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg=="
+    },
+    "node_modules/@sindresorhus/is": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.0.2.tgz",
+      "integrity": "sha512-d9xRovfKNz1SKieM0qJdO+PQonjnnIfSNWfHYnBSJ9hkjm0ZPw6HlxscDXYstp3z+7V2GOFHc+J0CYrYTjqCJw==",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/is?sponsor=1"
+      }
+    },
+    "node_modules/@szmarczak/http-timer": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
+      "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
+      "dependencies": {
+        "defer-to-connect": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=14.16"
+      }
+    },
     "node_modules/@tootallnate/once": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
@@ -35,6 +63,11 @@
         "node": ">= 10"
       }
     },
+    "node_modules/@types/http-cache-semantics": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
+      "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA=="
+    },
     "node_modules/abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -55,6 +88,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/adm-zip": {
+      "version": "0.5.16",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
+      "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
+      "engines": {
+        "node": ">=12.0"
+      }
+    },
     "node_modules/agent-base": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz",
@@ -184,6 +225,37 @@
         "node": ">=8"
       }
     },
+    "node_modules/browserslist": {
+      "version": "4.25.0",
+      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.0.tgz",
+      "integrity": "sha512-PJ8gYKeS5e/whHBh8xrwYK+dAvEj7JXtz6uTucnMRB8OiGTsKccFekoRrjajPBHV8oOY+2tI4uxeceSimKwMFA==",
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/browserslist"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ],
+      "dependencies": {
+        "caniuse-lite": "^1.0.30001718",
+        "electron-to-chromium": "^1.5.160",
+        "node-releases": "^2.0.19",
+        "update-browserslist-db": "^1.1.3"
+      },
+      "bin": {
+        "browserslist": "cli.js"
+      },
+      "engines": {
+        "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
+      }
+    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -193,6 +265,31 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/cacheable-lookup": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz",
+      "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==",
+      "engines": {
+        "node": ">=14.16"
+      }
+    },
+    "node_modules/cacheable-request": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-12.0.1.tgz",
+      "integrity": "sha512-Yo9wGIQUaAfIbk+qY0X4cDQgCosecfBe3V9NSyeY4qPC2SAkbCS4Xj79VP8WOzitpJUZKc/wsRCYF5ariDIwkg==",
+      "dependencies": {
+        "@types/http-cache-semantics": "^4.0.4",
+        "get-stream": "^9.0.1",
+        "http-cache-semantics": "^4.1.1",
+        "keyv": "^4.5.4",
+        "mimic-response": "^4.0.0",
+        "normalize-url": "^8.0.1",
+        "responselike": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/call-bind-apply-helpers": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
@@ -222,6 +319,36 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/callsites": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.2.0.tgz",
+      "integrity": "sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==",
+      "engines": {
+        "node": ">=12.20"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/caniuse-lite": {
+      "version": "1.0.30001722",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001722.tgz",
+      "integrity": "sha512-DCQHBBZtiK6JVkAGw7drvAMK0Q0POD/xZvEmDp6baiMMP6QXXk9HpD6mNYBZWhOPG6LvIDb82ITqtWjhDckHCA==",
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ]
+    },
     "node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -407,6 +534,39 @@
       "integrity": "sha512-8vDa8Qxvr/+d94hSh5P3IJwI5t8/c0KsMp+g8bNw9cY2icONa5aPfvKeieW1WlG0WQYwwhJ7mjui2xtiePQSXw==",
       "license": "MIT"
     },
+    "node_modules/decompress-response": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+      "dependencies": {
+        "mimic-response": "^3.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/decompress-response/node_modules/mimic-response": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+      "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/defer-to-connect": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
+      "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==",
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -448,6 +608,31 @@
         "node": ">=12"
       }
     },
+    "node_modules/dot-prop": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-7.2.0.tgz",
+      "integrity": "sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==",
+      "dependencies": {
+        "type-fest": "^2.11.2"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/dot-prop/node_modules/type-fest": {
+      "version": "2.19.0",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz",
+      "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==",
+      "engines": {
+        "node": ">=12.20"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/dotenv": {
       "version": "16.5.0",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz",
@@ -480,6 +665,11 @@
       "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
       "license": "MIT"
     },
+    "node_modules/electron-to-chromium": {
+      "version": "1.5.166",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.166.tgz",
+      "integrity": "sha512-QPWqHL0BglzPYyJJ1zSSmwFFL6MFXhbACOCcsCdUMCkzPdS9/OIBVxg516X/Ado2qwAq8k0nJJ7phQPCqiaFAw=="
+    },
     "node_modules/encodeurl": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
@@ -546,6 +736,14 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/escalade": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
+      "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/escape-html": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -696,6 +894,14 @@
         "node": ">= 6"
       }
     },
+    "node_modules/form-data-encoder": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.1.0.tgz",
+      "integrity": "sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==",
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/formdata-polyfill": {
       "version": "4.0.10",
       "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
@@ -750,6 +956,15 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/generative-bayesian-network": {
+      "version": "2.1.66",
+      "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.66.tgz",
+      "integrity": "sha512-gbBsyaaEJj/LHp3473TQrMDdcKiRzI8Sn2CbcG/lwONZkp0n9/ChC1mjzcbZQtHHCuqjn+JouSSbzLeepUMbuw==",
+      "dependencies": {
+        "adm-zip": "^0.5.9",
+        "tslib": "^2.4.0"
+      }
+    },
     "node_modules/get-intrinsic": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
@@ -787,6 +1002,21 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/get-stream": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz",
+      "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==",
+      "dependencies": {
+        "@sec-ant/readable-stream": "^0.4.1",
+        "is-stream": "^4.0.1"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/glob-parent": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
@@ -812,6 +1042,47 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/got": {
+      "version": "14.4.7",
+      "resolved": "https://registry.npmjs.org/got/-/got-14.4.7.tgz",
+      "integrity": "sha512-DI8zV1231tqiGzOiOzQWDhsBmncFW7oQDH6Zgy6pDPrqJuVZMtoSgPLLsBZQj8Jg4JFfwoOsDA8NGtLQLnIx2g==",
+      "dependencies": {
+        "@sindresorhus/is": "^7.0.1",
+        "@szmarczak/http-timer": "^5.0.1",
+        "cacheable-lookup": "^7.0.0",
+        "cacheable-request": "^12.0.1",
+        "decompress-response": "^6.0.0",
+        "form-data-encoder": "^4.0.2",
+        "http2-wrapper": "^2.2.1",
+        "lowercase-keys": "^3.0.0",
+        "p-cancelable": "^4.0.1",
+        "responselike": "^3.0.0",
+        "type-fest": "^4.26.1"
+      },
+      "engines": {
+        "node": ">=20"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/got?sponsor=1"
+      }
+    },
+    "node_modules/got-scraping": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/got-scraping/-/got-scraping-4.1.1.tgz",
+      "integrity": "sha512-MbT+NMMU4VgvOg2tFIPOSIrMfH986fm0LJ17RxBLKlyTs3gh3xIMETpe+zdPaXY7tH1j6YYeqtfG0TnVMx6V2g==",
+      "dependencies": {
+        "got": "^14.2.1",
+        "header-generator": "^2.1.41",
+        "http2-wrapper": "^2.2.0",
+        "mimic-response": "^4.0.0",
+        "ow": "^1.1.1",
+        "quick-lru": "^7.0.0",
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -861,6 +1132,71 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/header-generator": {
+      "version": "2.1.66",
+      "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.66.tgz",
+      "integrity": "sha512-g0jd79o0CyzyK0Jega4pAG1eJhykhPNfBLpOnUINtX2YkToVvRSBZ+B2wtmIjqwKHXK8DNWxylKuXnZmLs1yMQ==",
+      "dependencies": {
+        "browserslist": "^4.21.1",
+        "generative-bayesian-network": "^2.1.66",
+        "ow": "^0.28.1",
+        "tslib": "^2.4.0"
+      },
+      "engines": {
+        "node": ">=16.0.0"
+      }
+    },
+    "node_modules/header-generator/node_modules/@sindresorhus/is": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz",
+      "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/is?sponsor=1"
+      }
+    },
+    "node_modules/header-generator/node_modules/callsites": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+      "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/header-generator/node_modules/dot-prop": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz",
+      "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==",
+      "dependencies": {
+        "is-obj": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/header-generator/node_modules/ow": {
+      "version": "0.28.2",
+      "resolved": "https://registry.npmjs.org/ow/-/ow-0.28.2.tgz",
+      "integrity": "sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==",
+      "dependencies": {
+        "@sindresorhus/is": "^4.2.0",
+        "callsites": "^3.1.0",
+        "dot-prop": "^6.0.1",
+        "lodash.isequal": "^4.5.0",
+        "vali-date": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/html-encoding-sniffer": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
@@ -873,6 +1209,11 @@
         "node": ">=12"
       }
     },
+    "node_modules/http-cache-semantics": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz",
+      "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="
+    },
     "node_modules/http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -938,6 +1279,29 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
+    "node_modules/http2-wrapper": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz",
+      "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==",
+      "dependencies": {
+        "quick-lru": "^5.1.1",
+        "resolve-alpn": "^1.2.0"
+      },
+      "engines": {
+        "node": ">=10.19.0"
+      }
+    },
+    "node_modules/http2-wrapper/node_modules/quick-lru": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
+      "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/https-proxy-agent": {
       "version": "7.0.6",
       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
@@ -1054,12 +1418,31 @@
         "node": ">=0.12.0"
       }
     },
+    "node_modules/is-obj": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
+      "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/is-potential-custom-element-name": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
       "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
       "license": "MIT"
     },
+    "node_modules/is-stream": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz",
+      "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/jsdom": {
       "version": "22.1.0",
       "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-22.1.0.tgz",
@@ -1150,6 +1533,36 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
+    "node_modules/json-buffer": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+      "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
+    },
+    "node_modules/keyv": {
+      "version": "4.5.4",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
+      "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
+      "dependencies": {
+        "json-buffer": "3.0.1"
+      }
+    },
+    "node_modules/lodash.isequal": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
+      "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==",
+      "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead."
+    },
+    "node_modules/lowercase-keys": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
+      "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/math-intrinsics": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@@ -1219,6 +1632,17 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/mimic-response": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
+      "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/minimatch": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -1285,6 +1709,11 @@
         "url": "https://opencollective.com/node-fetch"
       }
     },
+    "node_modules/node-releases": {
+      "version": "2.0.19",
+      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
+      "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw=="
+    },
     "node_modules/nodemon": {
       "version": "3.1.10",
       "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.10.tgz",
@@ -1349,6 +1778,17 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/normalize-url": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.2.tgz",
+      "integrity": "sha512-Ee/R3SyN4BuynXcnTaekmaVdbDAEiNrHqjQIA37mHU8G9pf7aaAD4ZX3XjBLo6rsdcxA/gtkcNYZLt30ACgynw==",
+      "engines": {
+        "node": ">=14.16"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/nwsapi": {
       "version": "2.2.20",
       "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.20.tgz",
@@ -1379,6 +1819,43 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/ow": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/ow/-/ow-1.1.1.tgz",
+      "integrity": "sha512-sJBRCbS5vh1Jp9EOgwp1Ws3c16lJrUkJYlvWTYC03oyiYVwS/ns7lKRWow4w4XjDyTrA2pplQv4B2naWSR6yDA==",
+      "dependencies": {
+        "@sindresorhus/is": "^5.3.0",
+        "callsites": "^4.0.0",
+        "dot-prop": "^7.2.0",
+        "lodash.isequal": "^4.5.0",
+        "vali-date": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=14.16"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/ow/node_modules/@sindresorhus/is": {
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz",
+      "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==",
+      "engines": {
+        "node": ">=14.16"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/is?sponsor=1"
+      }
+    },
+    "node_modules/p-cancelable": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz",
+      "integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==",
+      "engines": {
+        "node": ">=14.16"
+      }
+    },
     "node_modules/parse5": {
       "version": "7.3.0",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
@@ -1406,6 +1883,11 @@
       "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==",
       "license": "MIT"
     },
+    "node_modules/picocolors": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
+      "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="
+    },
     "node_modules/picomatch": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
@@ -1531,6 +2013,17 @@
       "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
       "license": "MIT"
     },
+    "node_modules/quick-lru": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.0.1.tgz",
+      "integrity": "sha512-kLjThirJMkWKutUKbZ8ViqFc09tDQhlbQo2MNuVeLWbRauqYP96Sm6nzlQ24F0HFjUNZ4i9+AgldJ9H6DZXi7g==",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/range-parser": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
@@ -1574,6 +2067,25 @@
       "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
       "license": "MIT"
     },
+    "node_modules/resolve-alpn": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
+      "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g=="
+    },
+    "node_modules/responselike": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",
+      "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",
+      "dependencies": {
+        "lowercase-keys": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=14.16"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/rrweb-cssom": {
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz",
@@ -1863,6 +2375,11 @@
         "node": ">=14"
       }
     },
+    "node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
+    },
     "node_modules/tunnel": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
@@ -1871,6 +2388,17 @@
         "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
       }
     },
+    "node_modules/type-fest": {
+      "version": "4.41.0",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz",
+      "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==",
+      "engines": {
+        "node": ">=16"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/type-is": {
       "version": "1.6.18",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@@ -1909,6 +2437,35 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/update-browserslist-db": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz",
+      "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==",
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/browserslist"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ],
+      "dependencies": {
+        "escalade": "^3.2.0",
+        "picocolors": "^1.1.1"
+      },
+      "bin": {
+        "update-browserslist-db": "cli.js"
+      },
+      "peerDependencies": {
+        "browserslist": ">= 4.21.0"
+      }
+    },
     "node_modules/url-parse": {
       "version": "1.5.10",
       "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
@@ -1928,6 +2485,14 @@
         "node": ">= 0.4.0"
       }
     },
+    "node_modules/vali-date": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/vali-date/-/vali-date-1.0.0.tgz",
+      "integrity": "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",

package.json

@@ -26,6 +26,7 @@
     "chalk": "^4.1.2",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
+    "got-scraping": "^4.1.1",
     "https-proxy-agent": "^7.0.2",
     "jsdom": "^22.1.0",
     "node-fetch": "^3.3.2",

src/lightweight-client.js

@@ -14,6 +14,7 @@ import {
 import { proxyPool } from './ProxyPool.js';
 import { proxyServer } from './ProxyServer.js';
 import { cookieManager } from './CookieManager.js';
+import { gotScraping } from 'got-scraping';
 
 // 获取当前文件的目录路径
 const __filename = fileURLToPath(import.meta.url);
@@ -227,460 +228,82 @@ async function streamNotionResponse(notionRequestBody) {
   const timeoutId = setTimeout(() => {
     logger.warning(`请求超时，30秒内未收到响应`);
     try {
-      // 发送结束消息
-      const endChunk = new ChatCompletionChunk({
-        choices: [
-          new Choice({
-            delta: new ChoiceDelta({ content: "请求超时，未收到Notion响应。" }),
-            finish_reason: "timeout"
-          })
-        ]
-      });
-      stream.write(`data: ${JSON.stringify(endChunk)}\n\n`);
-      stream.write('data: [DONE]\n\n');
-      stream.end();
-    } catch (error) {
-      logger.error(`发送超时消息时出错: ${error}`);
-      stream.end();
-    }
-  }, 30000); // 30秒超时
-  
-  // 启动fetch处理
-  fetchNotionResponse(
-    stream,
-    notionRequestBody,
-    headers,
-    NOTION_API_URL,
-    currentCookieData.cookie,
-    timeoutId
-  ).catch((error) => {
-    logger.error(`流处理出错: ${error}`);
-    clearTimeout(timeoutId);  // 清除超时计时器
-    
-    try {
-      // 发送错误消息
-      const errorChunk = new ChatCompletionChunk({
-        choices: [
-          new Choice({
-            delta: new ChoiceDelta({ content: `处理请求时出错: ${error.message}` }),
-            finish_reason: "error"
-          })
-        ]
-      });
-      stream.write(`data: ${JSON.stringify(errorChunk)}\n\n`);
-      stream.write('data: [DONE]\n\n');
+      if (!responseReceived) {
+        stream.end();
+      }
     } catch (e) {
-      logger.error(`发送错误消息时出错: ${e}`);
-    } finally {
-      stream.end();
+      logger.error(`关闭超时流时出错: ${e.message}`);
     }
-  });
-  
-  return stream;
-}
+  }, 30000);
 
-// 使用fetch调用Notion API并处理流式响应
-async function fetchNotionResponse(chunkQueue, notionRequestBody, headers, notionApiUrl, notionCookie, timeoutId) {
   let responseReceived = false;
-  let dom = null;
   
   try {
-    // 创建JSDOM实例模拟浏览器环境
-    dom = new JSDOM("", {
-      url: "https://www.notion.so",
-      referrer: "https://www.notion.so/chat",
-      contentType: "text/html",
-      includeNodeLocations: true,
-      storageQuota: 10000000,
-      pretendToBeVisual: true,
-      userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
-    });
-    
-    // 设置全局对象
-    const { window } = dom;
-    
-    // 使用更安全的方式设置全局对象
-    try {
-      if (!global.window) {
-        global.window = window;
-      }
-      
-      if (!global.document) {
-        global.document = window.document;
-      }
-      
-      // 安全地设置navigator
-      if (!global.navigator) {
-        try {
-          Object.defineProperty(global, 'navigator', {
-            value: window.navigator,
-            writable: true,
-            configurable: true
-          });
-        } catch (navError) {
-          logger.warning(`无法设置navigator: ${navError.message}，继续执行`);
-          // 继续执行，不会中断流程
-        }
-      }
-    } catch (globalError) {
-      logger.warning(`设置全局对象时出错: ${globalError.message}`);
-    }
-    
-    // 设置cookie
-    document.cookie = notionCookie;
-    
-    // 创建fetch选项
-    const fetchOptions = {
+    const notionApiUrl = getNotionApiUrl(currentCookieData.userId);
+    const notionCookie = cookieManager.getCookieString(currentCookieData.userId);
+
+    // 配置 got-scraping
+    const options = {
       method: 'POST',
-      headers: {
-        ...headers,
-        'user-agent': window.navigator.userAgent,
-        'Cookie': notionCookie
-      },
+      headers: headers,
       body: JSON.stringify(notionRequestBody),
+      isStream: true,
+      cookieJar: await cookieManager.getGotCookieJar(notionCookie),
+      timeout: { request: 30000 },
+      http2: true, // 使用HTTP/2，这对于模拟浏览器很重要
+      headerGeneratorOptions: { // 伪造浏览器头
+          browsers: ['chrome'],
+          devices: ['desktop'],
+          locales: ['en-US'],
+          operatingSystems: ['windows'],
+      },
     };
-    
-    // 添加代理配置（如果有）
-    if (USE_NATIVE_PROXY_POOL) {
-      proxy = proxyPool.getProxy();
-      if (proxy !== null)
-      {
-        const { HttpsProxyAgent } = await import('https-proxy-agent');
-        fetchOptions.agent = new HttpsProxyAgent(proxy.full);
-        logger.info(`使用代理: ${proxy.full}`);
-      }
-      else{
-        logger.warning(`没有可用代理`);
-      }
-    } else if(PROXY_URL) {
-      const { HttpsProxyAgent } = await import('https-proxy-agent');
-      fetchOptions.agent = new HttpsProxyAgent(PROXY_URL);
+
+    // 如果配置了代理，则使用它
+    if (PROXY_URL) {
+      options.proxyUrl = PROXY_URL;
       logger.info(`使用代理: ${PROXY_URL}`);
     }
-    let response = null;
-    // 发送请求
-    if (ENABLE_PROXY_SERVER){
-      response = await fetch('http://127.0.0.1:10655/proxy', {
-        method: 'POST',
-        body: JSON.stringify({
-          method: 'POST',
-          url: notionApiUrl,
-          headers: fetchOptions.headers,
-          body: fetchOptions.body,
-          stream:true
-        }),
-      });
-    }else{
-      response = await fetch(notionApiUrl, fetchOptions);
-    }
 
-    // 检查是否收到401错误（未授权）
-    if (response.status === 401) {
-      logger.error(`收到401未授权错误，cookie可能已失效`);
-      // 标记当前cookie为无效
-      cookieManager.markAsInvalid(currentCookieData.userId);
-      // 尝试获取下一个cookie
-      currentCookieData = cookieManager.getNext();
-      
-      if (!currentCookieData) {
-        throw new Error('所有cookie均已失效，无法继续请求');
-      }
-      
-      // 使用新cookie重新构建请求体
-      const newRequestBody = buildNotionRequest({
-        model: notionRequestBody.transcript[0]?.value?.model || '',
-        messages: [] // 这里应该根据实际情况重构消息
-      });
-      
-      // 使用新cookie重试请求
-      return fetchNotionResponse(
-        chunkQueue,
-        newRequestBody,
-        {
-          ...headers,
-          'x-notion-active-user-header': currentCookieData.userId,
-          'x-notion-space-id': currentCookieData.spaceId
-        },
-        notionApiUrl,
-        currentCookieData.cookie,
-        timeoutId
-      );
-    }
-    
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-    
-    // 处理流式响应
-    if (!response.body) {
-      throw new Error("Response body is null");
-    }
-    
-    // 创建流读取器
-    const reader = response.body;
-    let buffer = '';
-    
-    // 处理数据块
-    reader.on('data', (chunk) => {
-      try {
-        // 标记已收到响应
-        if (!responseReceived) {
-          responseReceived = true;
-          logger.info(`已连接Notion API`);
-          clearTimeout(timeoutId);  // 清除超时计时器
-        }
-        
-        // 解码数据
-        const text = chunk.toString('utf8');
-        buffer += text;
-        
-        // 按行分割并处理完整的JSON对象
-        const lines = buffer.split('\n');
-        buffer = lines.pop() || ''; // 保留最后一行（可能不完整）
-        
-        for (const line of lines) {
-          if (!line.trim()) continue;
-          
-          try {
-            const jsonData = JSON.parse(line);
-            
-            // 提取内容
-            if (jsonData?.type === "markdown-chat" && typeof jsonData?.value === "string") {
-              const content = jsonData.value;
-              if (!content) continue;
-              
-              // 创建OpenAI格式的块
-              const chunk = new ChatCompletionChunk({
-                choices: [
-                  new Choice({
-                    delta: new ChoiceDelta({ content }),
-                    finish_reason: null
-                  })
-                ]
-              });
-              
-              // 添加到队列
-              const dataStr = `data: ${JSON.stringify(chunk)}\n\n`;
-              chunkQueue.write(dataStr);
-            } else if (jsonData?.recordMap) {
-              // 忽略recordMap响应
-            } else {
-              // 忽略其他类型响应
-            }
-          } catch (jsonError) {
-            logger.error(`解析JSON出错: ${jsonError}`);
-          }
-        }
-      } catch (error) {
-        logger.error(`处理数据块出错: ${error}`);
-      }
+    const responseStream = gotScraping(notionApiUrl, options);
+
+    // 监听流事件
+    fetchNotionResponse(stream, responseStream, timeoutId);
+  } catch (error) {
+    logger.error(`创建 got-scraping 请求时出错: ${error.message}`);
+    if (timeoutId) clearTimeout(timeoutId);
+    stream.end();
+  }
+  
+  return stream;
+}
+
+// 使用got-scraping的流处理函数
+async function fetchNotionResponse(chunkQueue, responseStream, timeoutId) {
+  let responseReceived = false;
+  try {
+    responseStream.on('data', (chunk) => {
+      responseReceived = true;
+      chunkQueue.write(chunk);
     });
-    
-    // 处理流结束
-    reader.on('end', () => {
-      try {
-        logger.info(`响应完成`);
-        if (cookieManager.getValidCount() > 1){
-          // 尝试切换到下一个cookie
-          currentCookieData = cookieManager.getNext();
-          logger.info(`切换到下一个cookie: ${currentCookieData.userId}`);
-        }
-        
-        // 如果没有收到任何响应，发送一个提示消息
-        if (!responseReceived) {
-          logger.warning(`未从Notion收到内容响应,请更换ip重试`);
-          if (USE_NATIVE_PROXY_POOL) {
-            proxyPool.removeProxy(proxy.ip, proxy.port);
-          }
-          
-          const noContentChunk = new ChatCompletionChunk({
-            choices: [
-              new Choice({
-                delta: new ChoiceDelta({ content: "未从Notion收到内容响应,请更换ip重试。" }),
-                finish_reason: "no_content"
-              })
-            ]
-          });
-          chunkQueue.write(`data: ${JSON.stringify(noContentChunk)}\n\n`);
-        }
-        
-        // 创建结束块
-        const endChunk = new ChatCompletionChunk({
-          choices: [
-            new Choice({
-              delta: new ChoiceDelta({ content: null }),
-              finish_reason: "stop"
-            })
-          ]
-        });
-        
-        // 添加到队列
-        chunkQueue.write(`data: ${JSON.stringify(endChunk)}\n\n`);
-        chunkQueue.write('data: [DONE]\n\n');
-        
-        // 清除超时计时器（如果尚未清除）
-        if (timeoutId) clearTimeout(timeoutId);
-        
-        // 清理全局对象
-        try {
-          if (global.window) delete global.window;
-          if (global.document) delete global.document;
-          
-          // 安全地删除navigator
-          if (global.navigator) {
-            try {
-              delete global.navigator;
-            } catch (navError) {
-              // 如果无法删除，尝试将其设置为undefined
-              try {
-                Object.defineProperty(global, 'navigator', {
-                  value: undefined,
-                  writable: true,
-                  configurable: true
-                });
-              } catch (defineError) {
-                logger.warning(`无法清理navigator: ${defineError.message}`);
-              }
-            }
-          }
-        } catch (cleanupError) {
-          logger.warning(`清理全局对象时出错: ${cleanupError.message}`);
-        }
-        
-        // 结束流
-        chunkQueue.end();
-      } catch (error) {
-        logger.error(`Error in stream end handler: ${error}`);
-        if (timeoutId) clearTimeout(timeoutId);
-        
-        // 清理全局对象
-        try {
-          if (global.window) delete global.window;
-          if (global.document) delete global.document;
-          
-          // 安全地删除navigator
-          if (global.navigator) {
-            try {
-              delete global.navigator;
-            } catch (navError) {
-              // 如果无法删除，尝试将其设置为undefined
-              try {
-                Object.defineProperty(global, 'navigator', {
-                  value: undefined,
-                  writable: true,
-                  configurable: true
-                });
-              } catch (defineError) {
-                logger.warning(`无法清理navigator: ${defineError.message}`);
-              }
-            }
-          }
-        } catch (cleanupError) {
-          logger.warning(`清理全局对象时出错: ${cleanupError.message}`);
-        }
-        
-        chunkQueue.end();
+
+    responseStream.on('end', () => {
+      if (timeoutId) clearTimeout(timeoutId);
+      if (!responseReceived) {
+        logger.warning('流已结束但未收到任何数据');
       }
+      chunkQueue.end();
     });
-    
-    // 处理错误
-    reader.on('error', (error) => {
-      logger.error(`Stream error: ${error}`);
+
+    responseStream.on('error', (error) => {
+      logger.error(`got-scraping 响应流错误: ${error.message}`);
       if (timeoutId) clearTimeout(timeoutId);
-      
-      // 清理全局对象
-      try {
-        if (global.window) delete global.window;
-        if (global.document) delete global.document;
-        
-        // 安全地删除navigator
-        if (global.navigator) {
-          try {
-            delete global.navigator;
-          } catch (navError) {
-            // 如果无法删除，尝试将其设置为undefined
-            try {
-              Object.defineProperty(global, 'navigator', {
-                value: undefined,
-                writable: true,
-                configurable: true
-              });
-            } catch (defineError) {
-              logger.warning(`无法清理navigator: ${defineError.message}`);
-            }
-          }
-        }
-      } catch (cleanupError) {
-        logger.warning(`清理全局对象时出错: ${cleanupError.message}`);
-      }
-      
-      try {
-        const errorChunk = new ChatCompletionChunk({
-          choices: [
-            new Choice({
-              delta: new ChoiceDelta({ content: `流读取错误: ${error.message}` }),
-              finish_reason: "error"
-            })
-          ]
-        });
-        chunkQueue.write(`data: ${JSON.stringify(errorChunk)}\n\n`);
-        chunkQueue.write('data: [DONE]\n\n');
-      } catch (e) {
-        logger.error(`Error sending error message: ${e}`);
-      } finally {
-        chunkQueue.end();
-      }
+      chunkQueue.end();
     });
   } catch (error) {
-    logger.error(`Notion API请��失败: ${error}`);
-    // 清理全局对象
-    try {
-      if (global.window) delete global.window;
-      if (global.document) delete global.document;
-      
-      // 安全地删除navigator
-      if (global.navigator) {
-        try {
-          delete global.navigator;
-        } catch (navError) {
-          // 如果无法删除，尝试将其设置为undefined
-          try {
-            Object.defineProperty(global, 'navigator', {
-              value: undefined,
-              writable: true,
-              configurable: true
-            });
-          } catch (defineError) {
-            logger.warning(`无法清理navigator: ${defineError.message}`);
-          }
-        }
-      }
-    } catch (cleanupError) {
-      logger.warning(`清理全局对象时出错: ${cleanupError.message}`);
-    }
-    
-    if (timeoutId) clearTimeout(timeoutId);
-    if (chunkQueue) chunkQueue.end();
-    
-    // 确保在错误情况下也触发流结束
-    try {
-      if (!responseReceived && chunkQueue) {
-        const errorChunk = new ChatCompletionChunk({
-          choices: [
-            new Choice({
-              delta: new ChoiceDelta({ content: `Notion API请求失败: ${error.message}` }),
-              finish_reason: "error"
-            })
-          ]
-        });
-        chunkQueue.write(`data: ${JSON.stringify(errorChunk)}\n\n`);
-        chunkQueue.write('data: [DONE]\n\n');
-      }
-    } catch (e) {
-      logger.error(`发送错误消息时出错: ${e}`);
-    }
-    
-    throw error; // 重新抛出错误以便上层捕获
+      logger.error(`处理 got-scraping 响应时出错: ${error.message}`);
+      if (timeoutId) clearTimeout(timeoutId);
+      chunkQueue.end();
   }
 }
 
@@ -688,12 +311,12 @@ async function fetchNotionResponse(chunkQueue, notionRequestBody, headers, notio
 async function initialize() {
   logger.info(`初始化Notion配置...`);
   
-  // 启动代理服务器
-  try {
-    await proxyServer.start();
-  } catch (error) {
-    logger.error(`启动代理服务器失败: ${error.message}`);
-  }
+  // 启动代理服务器 - 已禁用
+  // try {
+  //   await proxyServer.start();
+  // } catch (error) {
+  //   logger.error(`启动代理服务器失败: ${error.message}`);
+  // }
   
   // 初始化cookie管理器
   let initResult = false;
@@ -741,10 +364,10 @@ async function initialize() {
   logger.info(`当前使用的cookie对应的用户ID: ${currentCookieData.userId}`);
   logger.info(`当前使用的cookie对应的空间ID: ${currentCookieData.spaceId}`);
   
-  if (process.env.USE_NATIVE_PROXY_POOL === 'true') {
-    logger.info(`正在初始化本地代理池...`);
-    await proxyPool.initialize();
-  }
+  // if (process.env.USE_NATIVE_PROXY_POOL === 'true') {
+  //   logger.info(`正在初始化本地代理池...`);
+  //   await proxyPool.initialize();
+  // }
   
   INITIALIZED_SUCCESSFULLY = true;
 }