feat:fix the more detailed scraping

package-lock.json

@@ -11,7 +11,9 @@
         "axios": "^1.11.0",
         "cors": "^2.8.5",
         "express": "^5.1.0",
-        "puppeteer": "^24.16.2"
+        "puppeteer": "^24.16.2",
+        "puppeteer-extra": "^3.3.6",
+        "puppeteer-extra-plugin-stealth": "^2.11.2"
       },
       "devDependencies": {
         "nodemon": "^3.1.10",
@@ -69,6 +71,21 @@
       "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==",
       "license": "MIT"
     },
+    "node_modules/@types/debug": {
+      "version": "4.1.12",
+      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
+      "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/ms": "*"
+      }
+    },
+    "node_modules/@types/ms": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz",
+      "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==",
+      "license": "MIT"
+    },
     "node_modules/@types/node": {
       "version": "24.3.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-24.3.0.tgz",
@@ -155,6 +172,15 @@
       "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
       "license": "Python-2.0"
     },
+    "node_modules/arr-union": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
+      "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/ast-types": {
       "version": "0.13.4",
       "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz",
@@ -194,7 +220,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/bare-events": {
@@ -315,7 +340,6 @@
       "version": "1.1.12",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
       "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
@@ -443,6 +467,22 @@
         "node": ">=12"
       }
     },
+    "node_modules/clone-deep": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz",
+      "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==",
+      "license": "MIT",
+      "dependencies": {
+        "for-own": "^0.1.3",
+        "is-plain-object": "^2.0.1",
+        "kind-of": "^3.0.2",
+        "lazy-cache": "^1.0.3",
+        "shallow-clone": "^0.1.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/color-convert": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -477,7 +517,6 @@
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
-      "dev": true,
       "license": "MIT"
     },
     "node_modules/content-disposition": {
@@ -599,6 +638,15 @@
         }
       }
     },
+    "node_modules/deepmerge": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
+      "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/degenerator": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz",
@@ -970,6 +1018,27 @@
         }
       }
     },
+    "node_modules/for-in": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
+      "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/for-own": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz",
+      "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==",
+      "license": "MIT",
+      "dependencies": {
+        "for-in": "^1.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/form-data": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
@@ -1032,6 +1101,26 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/fs-extra": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
+      "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
+      "license": "MIT",
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+      "license": "ISC"
+    },
     "node_modules/fsevents": {
       "version": "2.3.3",
       "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@@ -1131,6 +1220,27 @@
         "node": ">= 14"
       }
     },
+    "node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "deprecated": "Glob versions prior to v9 are no longer supported",
+      "license": "ISC",
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/glob-parent": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
@@ -1156,6 +1266,12 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/graceful-fs": {
+      "version": "4.2.11",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
+      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
+      "license": "ISC"
+    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -1291,6 +1407,17 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+      "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
+      "license": "ISC",
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -1334,6 +1461,21 @@
         "node": ">=8"
       }
     },
+    "node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+      "license": "MIT"
+    },
+    "node_modules/is-extendable": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
+      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-extglob": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
@@ -1376,6 +1518,18 @@
         "node": ">=0.12.0"
       }
     },
+    "node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "license": "MIT",
+      "dependencies": {
+        "isobject": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-promise": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz",
@@ -1389,6 +1543,15 @@
       "dev": true,
       "license": "ISC"
     },
+    "node_modules/isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -1413,6 +1576,39 @@
       "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
       "license": "MIT"
     },
+    "node_modules/jsonfile": {
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz",
+      "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==",
+      "license": "MIT",
+      "dependencies": {
+        "universalify": "^2.0.0"
+      },
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
+    "node_modules/kind-of": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+      "license": "MIT",
+      "dependencies": {
+        "is-buffer": "^1.1.5"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/lazy-cache": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz",
+      "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@@ -1452,6 +1648,20 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/merge-deep": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz",
+      "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
+      "license": "MIT",
+      "dependencies": {
+        "arr-union": "^3.1.0",
+        "clone-deep": "^0.2.4",
+        "kind-of": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/merge-descriptors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz",
@@ -1489,7 +1699,6 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
       "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
-      "dev": true,
       "license": "ISC",
       "dependencies": {
         "brace-expansion": "^1.1.7"
@@ -1504,6 +1713,28 @@
       "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
       "license": "MIT"
     },
+    "node_modules/mixin-object": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz",
+      "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==",
+      "license": "MIT",
+      "dependencies": {
+        "for-in": "^0.1.3",
+        "is-extendable": "^0.1.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/mixin-object/node_modules/for-in": {
+      "version": "0.1.8",
+      "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz",
+      "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -1687,6 +1918,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/path-key": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@@ -1862,6 +2102,142 @@
         "node": ">=18"
       }
     },
+    "node_modules/puppeteer-extra": {
+      "version": "3.3.6",
+      "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz",
+      "integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/debug": "^4.1.0",
+        "debug": "^4.1.1",
+        "deepmerge": "^4.2.2"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "@types/puppeteer": "*",
+        "puppeteer": "*",
+        "puppeteer-core": "*"
+      },
+      "peerDependenciesMeta": {
+        "@types/puppeteer": {
+          "optional": true
+        },
+        "puppeteer": {
+          "optional": true
+        },
+        "puppeteer-core": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin": {
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz",
+      "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/debug": "^4.1.0",
+        "debug": "^4.1.1",
+        "merge-deep": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=9.11.2"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin-stealth": {
+      "version": "2.11.2",
+      "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz",
+      "integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "^4.1.1",
+        "puppeteer-extra-plugin": "^3.2.3",
+        "puppeteer-extra-plugin-user-preferences": "^2.4.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin-user-data-dir": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz",
+      "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "^4.1.1",
+        "fs-extra": "^10.0.0",
+        "puppeteer-extra-plugin": "^3.2.3",
+        "rimraf": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/puppeteer-extra-plugin-user-preferences": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz",
+      "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "^4.1.1",
+        "deepmerge": "^4.2.2",
+        "puppeteer-extra-plugin": "^3.2.3",
+        "puppeteer-extra-plugin-user-data-dir": "^2.4.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "peerDependencies": {
+        "playwright-extra": "*",
+        "puppeteer-extra": "*"
+      },
+      "peerDependenciesMeta": {
+        "playwright-extra": {
+          "optional": true
+        },
+        "puppeteer-extra": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/qs": {
       "version": "6.14.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz",
@@ -1932,6 +2308,22 @@
         "node": ">=4"
       }
     },
+    "node_modules/rimraf": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "deprecated": "Rimraf versions prior to v4 are no longer supported",
+      "license": "ISC",
+      "dependencies": {
+        "glob": "^7.1.3"
+      },
+      "bin": {
+        "rimraf": "bin.js"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/router": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz",
@@ -2029,6 +2421,42 @@
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
       "license": "ISC"
     },
+    "node_modules/shallow-clone": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz",
+      "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==",
+      "license": "MIT",
+      "dependencies": {
+        "is-extendable": "^0.1.1",
+        "kind-of": "^2.0.1",
+        "lazy-cache": "^0.2.3",
+        "mixin-object": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/shallow-clone/node_modules/kind-of": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz",
+      "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==",
+      "license": "MIT",
+      "dependencies": {
+        "is-buffer": "^1.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/shallow-clone/node_modules/lazy-cache": {
+      "version": "0.2.7",
+      "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz",
+      "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/shebang-command": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -2428,6 +2856,15 @@
       "license": "MIT",
       "optional": true
     },
+    "node_modules/universalify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
+      "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 10.0.0"
+      }
+    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

package.json

@@ -7,12 +7,14 @@
     "axios": "^1.11.0",
     "cors": "^2.8.5",
     "express": "^5.1.0",
-    "puppeteer": "^24.16.2"
+    "puppeteer": "^24.16.2",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
   },
   "devDependencies": {
     "nodemon": "^3.1.10",
-    "typescript": "^5.4.0",
-    "tsc-watch": "^6.2.0"
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.0"
   },
   "scripts": {
     "dev": "nodemon server.js",

server.js

@@ -1,19 +1,19 @@
 const express = require('express');
-const puppeteer = require('puppeteer');
+const puppeteerExtra = require('puppeteer-extra'); // NEW: For stealth
+const StealthPlugin = require('puppeteer-extra-plugin-stealth'); // NEW: Stealth plugin
 const cors = require('cors');
 const { EventEmitter } = require('events');
 
+puppeteerExtra.use(StealthPlugin()); // NEW: Enable stealth plugin
+
 const app = express();
 const port = 7860;
 
 app.use(cors());
 app.use(express.json());
 
-// --- Progress Tracking and Job Storage ---
-
-// Stores the real-time progress of active jobs
+// --- Progress Tracking and Job Storage --- (Unchanged)
 const progressTrackers = new Map();
-// Stores the final state and result (PDF buffer or error) of jobs
 const downloadJobs = new Map();
 
 class ProgressTracker extends EventEmitter {
@@ -41,8 +41,7 @@ class ProgressTracker extends EventEmitter {
     }
 }
 
-
-// --- Puppeteer Logic (Unchanged) ---
+// --- Puppeteer Logic (Updated for Stealth and Reliability) ---
 const bypassCookiesAndRestrictions = async (page, progressTracker) => {
     progressTracker?.updateProgress(5, 'bypassing', 'Setting up cookie bypass...');
 
@@ -66,7 +65,7 @@ const bypassCookiesAndRestrictions = async (page, progressTracker) => {
         }
     }
 
-    // Step 2: Inject CSS to hide cookie banners immediately
+    // Step 2: Inject CSS to hide cookie banners immediately (Unchanged)
     await page.addStyleTag({
         content: `
             /* Hide all possible cookie banners */
@@ -107,7 +106,7 @@ const bypassCookiesAndRestrictions = async (page, progressTracker) => {
         `
     });
 
-    // Step 3: Inject JavaScript to handle dynamic cookie banners
+    // Step 3: Inject JavaScript to handle dynamic cookie banners (Unchanged)
     await page.evaluateOnNewDocument(() => {
         // Override common cookie consent functions
         window.cookieConsent = { accepted: true };
@@ -161,6 +160,7 @@ const bypassCookiesAndRestrictions = async (page, progressTracker) => {
     progressTracker?.updateProgress(10, 'bypassing', 'Cookie bypass configured successfully');
     return true;
 };
+
 const unblurContent = async (page, progressTracker) => {
     progressTracker?.updateProgress(15, 'unblurring', 'Removing content restrictions...');
 
@@ -219,6 +219,7 @@ const unblurContent = async (page, progressTracker) => {
 
     progressTracker?.updateProgress(20, 'unblurring', 'Content restrictions removed');
 };
+
 const applyPrintStyles = async (page, progressTracker) => {
     progressTracker?.updateProgress(85, 'styling', 'Applying print styles...');
 
@@ -228,16 +229,25 @@ const applyPrintStyles = async (page, progressTracker) => {
         style.id = "print-style-extension";
         style.innerHTML = `
             @page {
+                /* Set page size to A4 and remove default margins */
                 size: A4 portrait;
-                margin: 5mm;
+                margin: 0mm; 
             }
             @media print {
                 html, body {
+                    /* Ensure the body takes the full width and has no extra padding/margin */
+                    width: 210mm !important;
+                    height: auto !important;
                     margin: 0 !important;
                     padding: 0 !important;
                     overflow: visible !important;
+                    background: white !important;
+                    color: black !important;
                 }
+
+                /* Remove all unwanted elements like headers, footers, sidebars, etc. */
                 header, footer, nav, aside, .no-print, .ads, .sidebar, .premium-banner,
+                [class*="Header"], [class*="Footer"], [class*="Sidebar"], [id*="Header"],
                 .ViewerToolbar, .Layout_info-bar-wrapper__He0Ho, .Sidebar_sidebar-scrollable__kqeBZ,
                 .HeaderWrapper_header-wrapper__mCmf3, .Layout_visible-content-bottom-wrapper-sticky__yaaAB,
                 .Layout_bottom-section-wrapper__yBWWk, .Layout_footer-wrapper__bheJQ,
@@ -245,21 +255,32 @@ const applyPrintStyles = async (page, progressTracker) => {
                 .Layout_sidebar-wrapper__unavM, .Layout_is-open__9DQr4 {
                     display: none !important;
                 }
-                body {
-                    background: white !important;
-                    color: black !important;
-                }
+
+                /* Force all elements to have a transparent background and no shadow */
                 * {
                     box-shadow: none !important;
                     background: transparent !important;
+                    color: inherit !important;
                 }
-                .Viewer_document-wrapper__JPBWQ, .Viewer_document-wrapper__LXzoQ, .Viewer_document-wrapper__XsO4j, .page-content {
-                    display: flex !important;
-                    flex-direction: column !important;
+
+                /*
+                 * KEY FIX: Target the main document container.
+                 * Force it to be a block element, remove any transforms or max-widths,
+                 * and center it perfectly within the page.
+                 */
+                .Viewer_document-wrapper__JPBWQ, .Viewer_document-wrapper__LXzoQ, 
+                .Viewer_document-wrapper__XsO4j, .page-content, .document-viewer, #page-container {
+                    position: static !important;
+                    display: block !important;
                     width: 100% !important;
-                    max-width: 210mm !important;
-                    margin: 0 auto !important;
+                    max-width: none !important;
+                    margin: 0 auto !important; /* Center the content */
+                    padding: 5mm !important; /* Add a small margin for readability */
+                    box-sizing: border-box; /* Include padding in width calculation */
+                    transform: none !important;
                 }
+
+                /* Ensure individual pages and images within the document use the full width */
                 [data-page], .page, .document-page, img {
                     page-break-after: always !important;
                     page-break-inside: avoid !important;
@@ -267,6 +288,7 @@ const applyPrintStyles = async (page, progressTracker) => {
                     width: 100% !important;
                     max-width: 100% !important;
                     height: auto !important;
+                    display: block !important;
                 }
             }
         `;
@@ -275,13 +297,14 @@ const applyPrintStyles = async (page, progressTracker) => {
 
     progressTracker?.updateProgress(88, 'styling', 'Print styles applied successfully');
 };
+
 const studocuDownloader = async (url, options = {}, progressTracker = null) => {
     let browser;
     try {
         progressTracker?.updateProgress(0, 'initializing', 'Starting browser...');
 
-        console.log("🚀 Launching browser with stealth configuration...");
-        browser = await puppeteer.launch({
+        console.log("🚀 Launching browser with enhanced stealth configuration...");
+        browser = await puppeteerExtra.launch({ // UPDATED: Use puppeteerExtra
             headless: true,
             args: [
                 '--no-sandbox',
@@ -300,8 +323,10 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
                 '--disable-web-security',
                 '--disable-features=site-per-process',
                 '--disable-blink-features=AutomationControlled',
-                '--disable-extensions'
+                '--disable-extensions',
+                '--ignore-certificate-errors'
             ],
+            ignoreHTTPSErrors: true,
             timeout: 300000,
         });
 
@@ -309,9 +334,10 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
 
         progressTracker?.updateProgress(2, 'initializing', 'Configuring browser settings...');
 
-        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
-        await page.setViewport({ width: 794, height: 1122 });
+        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
+        await page.setViewport({ width: 794, height: 1122 }); // A4 size in pixels at 96 DPI
 
+        // NOTE: Stealth plugin handles most of this, but keeping for extra safety
         await page.evaluateOnNewDocument(() => {
             Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
             Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
@@ -321,11 +347,17 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
         // Set up cookie and content bypass
         await bypassCookiesAndRestrictions(page, progressTracker);
 
-        // Block unnecessary resources
+        // Block unnecessary resources (UPDATED: Always continue for 'document' to prevent navigation failures)
         await page.setRequestInterception(true);
         page.on('request', (req) => {
             const resourceType = req.resourceType();
-            const reqUrl = req.url();
+            const reqUrl = req.url().toLowerCase();
+
+            if (resourceType === 'document') {
+                req.continue();
+                return;
+            }
+
             if (
                 reqUrl.includes('doubleclick') ||
                 reqUrl.includes('googletagmanager') ||
@@ -337,7 +369,7 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
                 reqUrl.includes('mixpanel') ||
                 reqUrl.includes('onetrust') ||
                 reqUrl.includes('cookielaw') ||
-                (resourceType === 'other' && reqUrl.includes('track'))
+                (resourceType === 'other' && reqUrl.includes('/track/')) // UPDATED: More specific to avoid over-blocking
             ) {
                 req.abort();
             } else {
@@ -350,7 +382,7 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             progressTracker?.updateProgress(12, 'authenticating', 'Logging into StuDocu...');
 
             console.log("🔑 Logging in to StuDocu...");
-            await page.goto('https://www.studocu.com/en-us/login', { waitUntil: 'domcontentloaded', timeout: 60000 });
+            await page.goto('https://www.studocu.com/en-us/login', { waitUntil: 'domcontentloaded', timeout: 120000 });
             await page.waitForSelector('#email', { timeout: 15000 });
             await page.type('#email', options.email);
             await page.type('#password', options.password);
@@ -366,23 +398,28 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
             }
         }
 
-        progressTracker?.updateProgress(25, 'navigating', 'Navigating to document...');
+        progressTracker?.updateProgress(25, 'navigating', 'Navigating to homepage first for session setup...');
+        console.log(`📄 Navigating to homepage to simulate natural session...`);
+        await page.goto('https://www.studocu.com/en-us', { waitUntil: 'domcontentloaded', timeout: 150000 }); // NEW: Preliminary homepage visit
+        await new Promise(resolve => setTimeout(resolve, 3000)); // Short delay for session stabilization
+
+        progressTracker?.updateProgress(30, 'navigating', 'Navigating to document...');
         console.log(`📄 Navigating to ${url}...`);
 
         let navigationSuccess = false;
         let attempts = 0;
-        const maxAttempts = 3;
+        const maxAttempts = 5;
         while (!navigationSuccess && attempts < maxAttempts) {
             try {
                 attempts++;
-                progressTracker?.updateProgress(25 + (attempts * 5), 'navigating', `Navigation attempt ${attempts}/${maxAttempts}`);
+                progressTracker?.updateProgress(30 + (attempts * 5), 'navigating', `Navigation attempt ${attempts}/${maxAttempts}`);
                 console.log(`Navigation attempt ${attempts}/${maxAttempts}`);
-                await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
+                await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 150000 }); // UPDATED: Increased timeout to 2.5 min
                 navigationSuccess = true;
             } catch (e) {
                 console.log(`Navigation attempt ${attempts} failed:`, e.message);
                 if (attempts >= maxAttempts) throw e;
-                await new Promise(resolve => setTimeout(resolve, 5000));
+                await new Promise(resolve => setTimeout(resolve, 15000)); // UPDATED: Increased retry delay to 15s
             }
         }
 
@@ -512,7 +549,7 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
 
         const pdfBuffer = await page.pdf({
             printBackground: true,
-            preferCSSPageSize: true,
+            preferCSSPageSize: true, // Use the @page size
             displayHeaderFooter: false,
             timeout: 180000,
             scale: 1,
@@ -539,11 +576,7 @@ const studocuDownloader = async (url, options = {}, progressTracker = null) => {
     }
 };
 
-// --- API Routes ---
-
-/**
- * [NEW] Endpoint to start the download process and get a session ID.
- */
+// --- API Routes --- (Unchanged)
 app.post('/api/request-download', (req, res) => {
     const { url, email, password } = req.body;
     if (!url || !url.includes('studocu.com')) {
@@ -575,10 +608,6 @@ app.post('/api/request-download', (req, res) => {
         });
 });
 
-
-/**
- * [EXISTING] Endpoint for the client to poll for progress updates.
- */
 app.get('/api/progress/:sessionId', (req, res) => {
     const { sessionId } = req.params;
     const tracker = progressTrackers.get(sessionId);
@@ -608,10 +637,6 @@ app.get('/api/progress/:sessionId', (req, res) => {
     return res.status(404).json({ error: 'Session not found' });
 });
 
-
-/**
- * [NEW] Endpoint to download the final PDF file.
- */
 app.get('/api/download/:sessionId', (req, res) => {
     const { sessionId } = req.params;
     const job = downloadJobs.get(sessionId);
@@ -639,7 +664,6 @@ app.get('/api/download/:sessionId', (req, res) => {
     }
 });
 
-
 // --- Health and Info Endpoints (Unchanged) ---
 app.get('/health', (req, res) => {
     res.json({
@@ -652,14 +676,15 @@ app.get('/health', (req, res) => {
 
 app.get('/', (req, res) => {
     res.json({
-        message: '🚀 Enhanced StuDocu Downloader API v5.1 - Real-time Progress Tracking',
-        version: '5.1.0',
+        message: '🚀 Enhanced StuDocu Downloader API v5.2 - Real-time Progress Tracking with Stealth',
+        version: '5.2.0',
         features: [
             '🍪 Advanced cookie banner bypass',
             '🔓 Premium content unblurring',
             '🔑 Login support for full access',
             '📊 Real-time progress tracking via polling',
-            '📄 Clean PDF generation with print styles'
+            '📄 Clean PDF generation with print styles',
+            '🕵️ Enhanced stealth to evade bot detection'
         ],
         endpoints: {
             request: 'POST /api/request-download (body: {url, filename?, email?, password?})',
@@ -681,6 +706,6 @@ process.on('SIGINT', () => {
 });
 
 app.listen(port, () => {
-    console.log(`🚀 Enhanced StuDocu Downloader v5.1.0 running on http://localhost:${port}`);
-    console.log(`✨ Features: Real-time progress tracking and enhanced user feedback`);
+    console.log(`🚀 Enhanced StuDocu Downloader v5.2.0 running on http://localhost:${port}`);
+    console.log(`✨ Features: Real-time progress tracking, enhanced stealth, and user feedback`);
 });