Upload 33 files

Dockerfile

@@ -0,0 +1,83 @@
+FROM python:3.13-slim-bookworm AS builder
+
+# Build dummy packages to skip installing them and their dependencies
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends equivs \
+    && equivs-control libgl1-mesa-dri \
+    && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: libgl1-mesa-dri\nVersion: 99.0.0\nDescription: Dummy package for libgl1-mesa-dri\n' >> libgl1-mesa-dri \
+    && equivs-build libgl1-mesa-dri \
+    && mv libgl1-mesa-dri_*.deb /libgl1-mesa-dri.deb \
+    && equivs-control adwaita-icon-theme \
+    && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: adwaita-icon-theme\nVersion: 99.0.0\nDescription: Dummy package for adwaita-icon-theme\n' >> adwaita-icon-theme \
+    && equivs-build adwaita-icon-theme \
+    && mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
+
+FROM python:3.13-slim-bookworm
+
+# Copy dummy packages
+COPY --from=builder /*.deb /
+
+# Install dependencies and create flaresolverr user
+# You can test Chromium running this command inside the container:
+#    xvfb-run -s "-screen 0 1600x1200x24" chromium --no-sandbox
+# The error traces is like this: "*** stack smashing detected ***: terminated"
+# To check the package versions available you can use this command:
+#    apt-cache madison chromium
+WORKDIR /app
+    # Install dummy packages
+RUN dpkg -i /libgl1-mesa-dri.deb \
+    && dpkg -i /adwaita-icon-theme.deb \
+    # Install dependencies
+    && apt-get update \
+    && apt-get install -y --no-install-recommends chromium chromium-common chromium-driver xvfb dumb-init \
+        procps curl vim xauth \
+    # Remove temporary files and hardware decoding libraries
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
+    && rm -f /usr/lib/x86_64-linux-gnu/mfx/* \
+    # Create flaresolverr user
+    && useradd --home-dir /app --shell /bin/sh flaresolverr \
+    && mv /usr/bin/chromedriver chromedriver \
+    && chown -R flaresolverr:flaresolverr . \
+    # Create config dir
+    && mkdir /config \
+    && chown flaresolverr:flaresolverr /config
+
+VOLUME /config
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install -r requirements.txt \
+    # Remove temporary files
+    && rm -rf /root/.cache
+
+USER flaresolverr
+
+RUN mkdir -p "/app/.config/chromium/Crash Reports/pending"
+
+COPY src .
+COPY package.json ../
+
+EXPOSE 8191
+EXPOSE 8192
+
+# dumb-init avoids zombie chromium processes
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
+
+# Local build
+# docker build -t ngosang/flaresolverr:3.4.6 .
+# docker run -p 8191:8191 ngosang/flaresolverr:3.4.6
+
+# Multi-arch build
+# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+# docker buildx create --use
+# docker buildx build -t ngosang/flaresolverr:3.4.6 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
+#   add --push to publish in DockerHub
+
+# Test multi-arch build
+# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+# docker buildx create --use
+# docker buildx build -t ngosang/flaresolverr:3.4.6 --platform linux/arm/v7 --load .
+# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.4.6

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Diego Heras (ngosang / ngosang@hotmail.es)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

flaresolverr.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=FlareSolverr
+After=network.target
+
+[Service]
+SyslogIdentifier=flaresolverr
+Restart=always
+RestartSec=5
+Type=simple
+User=flaresolverr
+Group=flaresolverr
+Environment="LOG_LEVEL=info"
+Environment="CAPTCHA_SOLVER=none"
+WorkingDirectory=/opt/flaresolverr
+ExecStart=/opt/flaresolverr/flaresolverr
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

html_samples/cloudflare_captcha_hcaptcha_v1.html

@@ -0,0 +1,219 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="Just%20a%20moment_files/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '67839',
+                cRay: '732fbc436ab471ed',
+                cHash: 'dce5bd920f3aa51',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
+                    rm: 'R0VU',
+                    d: 'MqxNbbGfWazPaVMZ7GQRz02TV/pSUL9POWx0y4e7HFRwP1RTAxLc1RZRuHg+N/bGMuPj08kSx0UpcjEjMkSOqiU6I/64IDYbCJvey5rY07fkkljpZaYGTDZIoWdOWlgP3ky15ybZ42xMK4tfI1yJ+iFZCVgR6VBjJzi5I56j9Ijog2AvsoQW2TrguGpgKaT1LkhxWNElzBbvXWt1uyRgE19UQ9J/5vtxEwoh5wodHh7WE297n8uI1hpDgge2bDYQvwe+RDq3QAyhQOmymg+IIlt1y115v9R8k5ehT9TFY3vYvYnoJu9cOyHYprf9Z0jTNGxSTvLHYJbfq30Samu5fKfE0oZREZizvPUgUsJm2rRKkCY9VCdBkpO8vaUgIwIYkeWavtqdudjb3zEDBCD4cAH/xv3Bl1VRy2Qf7XlcbpElCOq06TDTQ1uGjyCqbVbvjesrOy0Dp2nXTjdfbkWvnN7mWpFlPUD7/41MUo9lc6V1Aj1Kjg6AKfVV4DUHpq6ZVnMHzrcPQLy4qD7CptcMpQKArZtJCRsUpgq8GWKJcU4dU8ZmyROAA+l+JEVnGbh2bsRdif4azh57OdjZfEKSa5c+AL3i66vyWAZCw9Wl6CAQdFTA+ixkbl8zKbCm8ulv',
+                    t: 'MTY1OTIwMTMxNi4zOTIwMDA=',
+                    m: '3l81qRkXiMTbjTzBtc0v1XwSheF46UfagbXVhYgbAVw=',
+                    i1: 'Iu5a1gH3p9igzqBwncow9g==',
+                    i2: 'PmNXozjc73unhnp/X0+kUQ==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="Just%20a%20moment_files/v1.js"></script>
+    <script type="text/javascript" src="Just%20a%20moment_files/api.js"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="Just%20a%20moment_files/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: block;">
+                <div id="cf-challenge-hcaptcha-wrapper" class="captcha-prompt spacer">
+                    <div style="display: none;" class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha.html"
+                            title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
+                            scrolling="no" data-hcaptcha-widget-id="0tiueg8lyuj" data-hcaptcha-response=""
+                            style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
+                            id="h-captcha-response-0tiueg8lyuj" name="h-captcha-response"
+                            style="display: none;"></textarea></div>
+                    <div class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha_002.html"
+                            title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
+                            scrolling="no" data-hcaptcha-widget-id="10tlmhzz0qyq" data-hcaptcha-response=""
+                            style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
+                            id="h-captcha-response-10tlmhzz0qyq" name="h-captcha-response"
+                            style="display: none;"></textarea></div>
+                </div>
+            </div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fbc436ab471ed')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">the first
+                    botnet in 2003 took over 500-1000 devices? Today, botnets take over millions of devices at
+                    once.</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: block; visibility: visible;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details">
+                    Requests from malicious bots can pose as legitimate traffic.
+                    Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="P4fDbSohR3e3VZmGdBSN0Gd8t8ueht.ZVgSdQYwa45Y-1659201316-0-AesEKnKN8eJLiLESJle3R0T3fwKbVMlX09CR0sIU1LruDXen0nSlT2a5OpMUFYR7HQMGcF9Ja227n2p2D2ffUlWHPVeFX-YSNiewLZA3XuAQmOn-1DyWKA-SaMH_MW2vOSC7PCHAdJDhoRWjM_o3MyKziopj3WmDcaCI_ikk68bJTIValZ_e9tO7hmHC8zjsxDC8kXmI0tbrhyW5nyS2hRlx_ZVRcRHbHsVRN0-FGtEbCoaHmnp-q0N4AYhCJXofYRunPcSG_Y1iWMk-7ofOXON_gO7oGG_8-WWD5EG1jaz2ldpNO1RTkS7dQvTiC1Io1qAsVnQtokEaDR2zoWK_MF-hz6tOmuJIDgnAoH6vPFAa9EyJOUiG2RV-3q1CKTUgr82XRJw5CaXpN0QeBq0xHxFl5mzkFO8xqQsRnPkGUKtxBQ58syPIhR4AvNp8HA028gUNmaztJZ9i2UcWydut4VghHsoJjS5DEKTamjJhNrrkargjXUekXTfKXMVKCXxo0NFObTmKwzsNB5hrk3M43KzZCOOgTnqsrVUk54bAeDsr4qmTVW2wVk-0u78QpV2JFFOIJxRLikPmqo9CUokgUJ_IPsEjA5Q3kjrf9yq2OHU0MkwzLFNOAyc5N3A4WSYp91kESwxM98qFetpAZ0R3LID2c2-MraHnpOI2Xn4bxbDIdUPmjy6VB8Huuuf6M-o3Tw">
+                <input type="hidden" name="r"
+                    value="bdZ7.nm8dGOZxq3EDOv_Kx7nKVv68q7b0RARXAlR9kQ-1659201316-0-AawyK3x4GgWasA2OtBBEp9Ea52qs8zEWwnQJxLWUnC+1jqlxaKHTIHeVQjvrTl/ccu6QA41yrSTKvazKiv6zQEiDj/6ziYkhldx+oJ7SqgMzPozzza1jofsGpCCPAzIlDicF+7sh4WKOxUJOeHgHCgfEZF/MPNsaahvbQ10U8Ei9tmvj8c2tkoybya75Bj5XHPPu0S9hnOH7S24ltm9vmyHlttI7uuI962FzPCTGjuAl4R/5+06WVAzBCJrS4biDNIuyYe22PtLl4b3Yf55eW7AFgyzKgddsohZJuNNliKyD6cusHDhm7MYpnXc5zwTdCbt6KGK/tBaylNyYwH/WBAUhyRYN5EVt9/iIKHrb+P6Z0RL4nO3BtQE/Zwx1VC3g1Wy4PPQJjqLixQptzl5eIzu43JIO/LBvT/mWuheH4eoPlghvyMYwfHcs7B4d7FCv1Tj9Skp9Fcj6HBAZlq/ss/eIwk7oOcTviQs+EUF9/yYatgtpXX9RCyvhMU6/ghOLfXRmOpAzsmoGnVqEpc2IMlZegYtieLveXU35cGJMI6wCR2ciCJIX995vLuL/4BdCAMEhyMAUWxtaCD2ZfRHyOWKNuf80w9k6/Ofhu7RevCr2mjQJAVTyE2OWWgOUuYJ4pZim93J7slMXieL3S5/JM08Q8g179Of7dzpN/oG7s80ljxAiCprpUAwpEmNiqNJN//v0e9KxknhCHeAWSAe8IeXbp5PSEQHXTmsqOFRkpud1pTsETcNbdonk8XMyv8mZRcFPVWRRWUb8hupn/d+x9r6mOdKdJkH8ZZ0R30LG0SLPYEvsVr2yU9o+uCZrRWkuE3SP3Lq3BIx+0vtm0DOvj6cODxy5/4Zm4x7LIpSa9wr69Rs2x+t+U5ydUupZ7oiAbWfYZSXHpmB0zJYOLMPJZcut50J/IgWuTMda8QBcTG3jRr4BTwpcmBZRmddfOJYgD7EMpOi1HgwLnS7l5QELafaMn0Hl6G774GVy4lEK2jURG9IEE3PV1m5Y903pqldFkJQsMxdisJWOzVjbtf41fxxnt4cQgiDQhktqCwg8xP6ijzPeWgvQHL5fMq61cQ5/4HB+yt9wKWMlBfUJR+ocI2MYx2nUWz+0BwCnTU29D9bx1xkir9bsnUnfOlfRDO2OEvI0iTe82666rVQO9XqTEz3POxrJYzLcSC9fTHpHfmCVwT2zWGGLi6pW5kqZh/uzSQ12MSvF5+dwvhe7yRks5gwMhnDHMQFyxKw3Xxm+dq2Ix/1uUucOhCu2L72j/NIwkF3Z7O7afY9nIu+NqMe8PbPJjq5ovEluosQfAMzWJH5Va8iur8o6K1y6hm7XFdNYAR+uCtxMw6WzF58QWVXXrDvfPeBMaNz+VVCnGP9elAwv62tc4Uh2SCbKbWdZchtLHgrJgYgQtCMhBDh0AXzE6ubbtfm9jE2vWcPj5jbo8U72i1pL2j8Xfr562Xc2WrQ7tKvSFQepGxfu2XgF7q55XKVqrnrBeXxZViUkB/gyXxI26CfrVfPLW+sYUo3JS+eCjyn2K7phv+630ixdpKrRJCTmkP3G8tcoLTJCB67/pbz+dXiNSB4JlHf4i3FVRkr8TAWS2zuMjJhB+ZyxnrGq/m7KSwpEEqgSCpOrQ5nkeoKIOyITfe9EPBSy9QtYDK+SAhUiLnICVURK7kGgrhZuKyK5/nyK9l7ffg16aaChJBisPBeiYsTDHlAeq0GbW7VR/jQDAVtVldeyD/dM5rJ4X+wl3A+faYD1OUxYT3n8dMs+E/1jLnYixXJpo7iXCqlTV3phOatg4XDQ5Bj6EYQIljVI4x2e8XHspcETIa0WepLsZF7WUtY7KbN8ZyDBFXgTMb4lzPmWyY8hZ05uX8EBKqUJhWh91AUob/OpJdf+u3axDDeRgsjl8K6CFM/5uQKo93co3KPqGZiqx0JoVj1t6KGxkrzYsgwlrTyeL44cEgr0zRQz5oFExuwHGYyogbHZ8EvU1eoiJ3IuQFxUH/1ULidfGtB371RYfz9gqONi1KiVzJ+zLjw+4HgMKXOV+ra09Dyg+eyUNfHillLXkhKWVoDpUhc+r46W6vXFp3oMKUWTRM0dE7iHofo+0tHb73d3ID2blRXgUeoMCQwOptoAYlFBIUYjggrIhd1AMC8TiZmiNULyP5imDePwcfq+ZjGH3o8VKRI2FcoFmChQegGco6pEbB5DxCguDuJbFRwGH4t9T0y74ZhlZiTNKA4xXsQnfIBEC5qz3mkcDAWoe73zqFAjp35JRVBjo3UDvehJppxzuoCXt9UbeuNEGll5/YJR4lfbUsEai0U6TFVleTTY53ofYCWEM6EnNDIToTFbm514YFTUSc4h8Qlq2fPeqC3IcCmirNT4Kf0FCO7MQrtGNFPJme2cpb/pZguS3pxxkKb4lOS+eGiUBGcSs1v3zHroJ+hum4wTJFRG0Yb99aCVQU44wgV3nKW7FZkXzwO3QY7nnkFI2kaAXerCPF4+Ho463g==">
+
+                <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
+                        1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fbc436ab471ed');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fbc436ab471ed';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="Just%20a%20moment_files/transparent.gif" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fbc436ab471ed</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+    <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
+        aria-hidden="true">
+        <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_003.html"
+                title="Main content of the hCaptcha challenge" scrolling="no"
+                style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
+        <div
+            style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
+        </div>
+        <div
+            style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
+            <div
+                style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
+            </div>
+            <div
+                style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
+            </div>
+        </div>
+    </div>
+    <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
+        aria-hidden="true">
+        <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_004.html"
+                title="Main content of the hCaptcha challenge" scrolling="no"
+                style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
+        <div
+            style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
+        </div>
+        <div
+            style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
+            <div
+                style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
+            </div>
+            <div
+                style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
+            </div>
+        </div>
+    </div>
+</body>
+
+</html>

html_samples/cloudflare_captcha_norobot_v1.html

@@ -0,0 +1,170 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="Just%20a%20moment2_files/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '94250',
+                cRay: '732fc1c74f757330',
+                cHash: '8c4978fa93c1751',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
+                    rm: 'R0VU',
+                    d: 'C4CtJo9JDMtUWZ0r+/s2CwjYdSTdqGYK3qFo1OXpvSc9v7/3d5QuMwmvG3e5oV1BpjlQb8eJJ23gVRxavjw/gpPp1brmKoHuvcJEmAP3Sof38vqcpF91/9NHe3JbmCM2xshiGvJdbpJXb5wXdYKYPMqy7NUHL1VU4hupa3Da3tBq9zyuMa1NcZaiyeE6piSl7n96m+VziRdwyG+SBUldIG/Fsv9J1yl+Gj19wbX1XEneMXChcClGgRrSe1MTd9thLkq2NGFqROnsUmpA8b+2Eqi+IPYQfkPcydWkHmJqQixN9ZFTIBChIC60hGHOQ7O354ju65tVGAhB/nBRREpdqvwoYzgufgg83+dbPHVdQasiuLRHvftOtHhS5/iaBOVoEBH+rElTSk/OYjU2Yh6gkQj0FjkbebEBptFeVAxgqoYZljOrhamWYYZ14tOKeonzc1rz/FXNTM5qVtrWCwAlt9SsXDjM/GYXZMTbOdNLnLZGlLNQCx+l6hMC0OQC45sWFzZECljbjXwiYfodKobeqe11lUXnskj8AN5Qc7O8OqtALsxoNCLZ7ou+ORY0lauremeuu3U3WqadgSGFGA+TZZw2VcCA3BIUKCGlsNLBlJ8wQS2UAGJfGLOVuhErmtsM',
+                    t: 'MTY1OTIwMTU0Mi4yOTUwMDA=',
+                    m: 'eWHHJ28v6yOyvSePVqcdyHxAYkkc3xq3VJ8YiDCk5nk=',
+                    i1: 'M3dMvem+HcwSbNQrJbaYdQ==',
+                    i2: 'ebY327qYCu6NZKHSQXkbaQ==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="Just%20a%20moment2_files/v1.js"></script>
+    <script type="text/javascript" src="Just%20a%20moment2_files/api.js"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="Just%20a%20moment2_files/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: block;">
+                <div id="cf-norobot-container" style="display: flex;"><input type="button" value="Verify you are human"
+                        class="big-button pow-button" style="cursor: pointer;"></div>
+            </div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fc1c74f757330')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">botnets can
+                    be used to shutdown popular websites?</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: block; visibility: visible;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details">
+                    Requests from malicious bots can pose as legitimate traffic.
+                    Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="UPeuijc1TS5ZQ21GIY6wjg6HHN_jWKH9sqolcSJABwg-1659201542-0-AR_ZxgiwVB4GwEgAjllIrmnGAumHNwuvfpFBddySYLh6CWexrUnxVYlX_wlB19Yndm45fs-KngMxbYB4dEOuf4MOJ_yL_BsNG3_cIPybV0bNn9WQXecJg3FfFrIBuMFIappZOX4hdDjLtRo9f4JsVsU6FzD9sUoKJRd4BTkjTAm25yFbqmPgV15XZhnJ5HRux044u0IIOVZCwTTzgRLCqToVb-OfiuUcHBzt4W7_wNlF1ObUi2oEr00DA1zZvzzY2KnXdZVN8m2OaNY_f2zkk9uDlLQRob_Ti6MHPNDr4eRkyMqZMZ1XDCxe-9lBkcEfpqtg6_4yac9ZiIEoNdJnJVE6cuNzb59DcBooXAq3IWp6fK4y4UIBStjqOXk4bxQb5yt1COfdPuQ9iLE_7yYOPG_t7n5I-4mjwvG7_U337A17oeEemXHfJkGC88Vm3SQdEHiW96VJuOA_X-rb7p3iOMlLYB5DKJ5DaBoPnP86uAWhoHWE6nrVzeAxeQ1y0uBHYPioJba5Kn9d-e2HsTMuAi7ZgSKuk90ApclIiW3owI4bLc4wxO5cu3ZIz7sZfbdvIKDhf9ESZhpQrITU_4Hgqjz0s3lt-MVeNP_0bz31XSeA--pdiulzUpQWLx1jhC4s7Av6STUb9bmbHpE41283KbbpuzBbmHN1UczNiaaquYZiEXRHKYyEMhKD782nWTJwQA">
+                <input type="hidden" name="r"
+                    value="i1ShtnCs9Zs8QexeFnp6EFtrWs3WbGEVQGXbVfYwpRI-1659201542-0-AbDM6G9qkbgoH+BqDdr1tzCDHr/DU9Sdxelapvp2/FZN6VqYfpDkJGv+HxhBQng6aVktcEobxp2ouOxJZxPQrR6tVFIhOW6uPOAdy5kh2BBJWUHfER13aq8LQ86fvDyRh3AThEHj6bgs2udacfvOrDrHT2j/KHBPePlGKbh8rzDTJBKw0ejUleHk8eKX/BQ1bVULgxT+ZZY721lyn2wrjsde1j1OAsiiCDkVvQ4Rs+Bas7UApD5HeWzyrCu2VFk/Qf+Rk+6spM+StYenQUAKXXrekJoIeNxPf/W9ZRsJfwUoY0JUK2thOWiwQOtw21nVDpiCFB9nhhOsmBzBoRQGjckZyu/O5U7jMIFdS9ThCFC0Kffg0MEr5xTkmgw+CNSwN7AlI9v3GS2XdTFOPXe29b68fZXzYfbm2CjqYhmxomZjCGTAmkzXWaVnMOs9Vl/8VurCUEu8SAt5k9Za/vFrEurX1edXNCviVuTOBSLHjqBiLui9FbufzGLq6BaHYi3WIFA1nMkoxduxbErP+Eqyi8UNvzvmEqUbj2COalXcQzkbHkyyLo33MNHZEi1zhhHjwCm1lp6mm4BRe60kRgTHb8X7oxBpY4vEcMz4jQQdsW15xBPAjsH8m9cj1H2ujpd7kfo8JGTyZ7FcoxOzGOuZr8XRpGkH72HaWYz7M+GIb3BBZ1v2Za7sSrzinNLFjHCCVXq68MqOmZ6RhgeexGoKJzcMHsHvgGXB8CisyyNTtA3OQOujybNUnNzlW7vJ/wDreTHkko6jQ/Lm/X2GnLg85BIg6IeROzt3eInAYsCaNKpST/h5bvSGCyzRoOW46oO8ZzZrV2FI2rEr0xLTIVWzQ//K2iGOCz58RisCfxWiF2n+fzj/5nE/0cjTPzYP68TM5BxB058EO7ZEFbgqhUji8IR9V2ahy7kI9dUhwd2S4IyjL+O6hCNPwjpRohkt93wXUCZDMgNoxi1BIylqqtAxYBfodyjFz8mB8GgcqBaCHN3tI0BINENVfvSJwKniYxL73frTX5KEqniT9GdT15o4F7QLf4S1atwYzF6ezJTYgLf6fOWUZKpaFMSRzEsxmZDmOFZeiss8lj7bKS6drOpkaOYzZiSgp5t5VwLKT0yQ+PDWQmqkpZ5WOa9/ayXLyOCunzk1IUO6VkvgFe0P2LZC9XEZUfwAFakYemej8/SZx0EknoPob1il3MMsbfHNAvcvUJK9xDbdAQ7rz34r4D5zO2aPnmYw1yv9K36z78I2dZpjVT9kpiKFwaOTkuSDUDtcmnhKM1XE+goG/C66G6PsChpGKLCeaDw4Rp7BxlumiSGB4Mp/bs8pTz3gez7pSu1oNodr7Tr1wJvCK8T5nVJ5GRO/tQ+Ff2K2s67udoV0CFtKufJyRsGCEv/0u5sArg3uwtwIz1W0JtAVjhe+J2nUihLa0Gqm7AwcCwfhsLHOhMG28V2NAw19iVq8RuMN7A2kGg5PH6bUeilWUxxZvWyDfyRSJZYMQytwAJdt4gQ++Qnl3mcaSk1N3pSiltVUDpfLcYb5gd35m+mKQWtPnIDlJMAtGoBeqROQPLNDg+LYdI/dnJzIOHjI3J+pTWhbAlF7B7NtccZOHmI9Cl3vS6Fpqs5aSPEDoDENTap6JN1kgm5NszMay9tAm66AcKF95W6QhwgQsyRrwScgRaPUtCx9ZJcbav6T/CAulcBB85MjwAd8+HF1g+UZT9VvChZoxh7NzfMoR53pVbxvW6acO8oVN5ITTP8mNAIisRvWi2KVdi4KqaLjYtLFNN8AMzjAC0vBIaFyGZlIbFsB44MRiMufD64b/66dqeC0l0WrUlUG/DgrnSQr6lgK2gONJKPQZGXoaK0Ga8O8xMOkaFLNaqH5UH5KpHvIQ8nwhuXk/MS/7Gdp1W02OEB4l0hhKFytgWdo9QmCquSatvOjuFyRPa6tV8ceGmuDnQw22bJM9BwzdKlHn/2/mHjCz7gcEA3Hb/CbeP8V8mF1mc5R8HEEdz/rx+BTESmGiRivv+WQYpRKNh77iqbYvCvkduK4b3UErNbvcS10aTt8zDFF/oIwjDpsniJsrIUcC0FdQRs2dqPIfkoSHvs7YGmOjx9QThCAiTkPKUE9C5C4YPY4CWRV3nYAFJrTq0F047PkzDYm0AJMCahWK7Vq/Ra3l3nRHu9yI+P0HiruUbkzLgiEJYnAuUtxvpC/Vj0uhr+A0R9Obs1MHkwtDuMs/ETh3ZymeFtWLj70StkslJxTzKGimZSsqQXRFYGHY6CHqHwIXGrArYNjTty48VIfbfaEu58KQp6roOdFmx90AcK2lV0V5UdyuzDJeH/V5ERAmxWLrXQKWgiDrY4ZqecnRk5XEAVMq/ChPts9gR7xsQK5WsHtQNKLfltkL8YvAoS+jZvxzfUUBg99YSC4J/HzQS+FQAnkDxCgeroahXysNN1bgDASXOrn3NsC3LYpUiZ2AVTLPkj1roR9r65O">
+
+                <span style="display: none;"><span style="display: none;" class="text-gray-600"
+                        data-translate="error">error code: 1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fc1c74f757330');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fc1c74f757330';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="Just%20a%20moment2_files/transparent.gif" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fc1c74f757330</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+</body>
+
+</html>

html_samples/cloudflare_init_v1.html

@@ -0,0 +1,120 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
+    <meta name="robots" content="noindex,nofollow" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet" />
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '46449',
+                cRay: '732fd3bc9c1d72de',
+                cHash: '8838fcad2a7f56c',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'Y3VybC83Ljg0LjA=',
+                    rm: 'R0VU',
+                    d: '+SdFLvm4kJf8Z9BVci1ZbUOY6ab/Dm5Zzyb0IvscIzmY9PnAAcvPfJ/3TD9YJViBxB/ArnbCQrOUfbSkq4odyaZmW19gm+exRuL8Z3POm1ABs7y6jwMshM19q4Gr3eFY/MUO/IYWuyA2F9q94hRCI6ZNb7dLEh9yh6hORbKRd62pdn59h1xCx8tNdKDtP7VXPXo85nYmJJPLOdXTnII+YxZ03a4isAmBHbi+lGoQN/bCV0K006VmpfPElAfAO9jm45o7pc1NgPQhZSKWpTyI/nHMueH6wacPREzN5RtREoQfKuwYpV++Gq56qr5bAe/SKeF+rI0x7OSqC4HQvrNwbA+kHZzaxgOKeiMFjDxmro/GyC/+sxeZmrxnSIAh4BScjPxEl1FLLkg/6D0JH6HmxoT8N/Jgpi9447Am4WeX+WQxJ9+uDs5WrFIahx7pWrgcZUTRPh+UCu3allJ2Q3cAfwK6BclhES/HhBBbJv0pnR1R2RfKDM/gr1MpLuhaK4mFEO/kSyNUjOnCjOfd+5d7Qb0DZn7sHpF2SVc+zNv5OWSvCRDUcNHjIOV6fq0datVyVWmxD6unPS0MMUFO+ZZNiB4ionrhVCiLrb2FjPQ8tzyCqXg+tnV7WtZ0h4+JuK3rxcaQ8PQy60/As8dKHqVTnw==',
+                    t: 'MTY1OTIwMjI3Ny44NjMwMDA=',
+                    m: 'zvAOPvfoONkW1BzH+jMnKOPtDpPpZijRP52DVDWH+i8=',
+                    i1: 'dDlQDNhOEuHzFEPo/etoAA==',
+                    i2: '+LTK9hchBRjTTQk1WQU1Vw==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'IdIU2i4FhVxxcYhzSFWdjoBuQm7qnyVK65JGofJuWV4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+</head>
+
+<body class="no-js">
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)" />
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fd3bc9c1d72de')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="DpGhFnuVRfDhqsQNASrgdT4WiiJ8m6lqTIs03.l6RLc-1659202277-0-AfUEAk9DsJ4rmpVI_Al7-eogy2CmM3YgWe4-31iw0oG2CcDIbYvauEW2IvK9m27_gq1FvdH-UPaGHR0q6Q2haXlX4pgmQK5rlQUSEd5HquGdtWMasHWqL_Q_TZGdOKz30bE2FEk8wLHRErHJRJloDRj0tiG8MreT2La_GLvovNK1XbMXDxFZT2Cc-DThBvxbgbDffw3okYfdl1ECXhLw9G6L4o8xgLsz3QZQG3dNZNhm5n4mf55-BBsFDzDTEN1_1BgORVw3mtbsodedktcACsVBCRupyBpTev9MML1jHzk06ZT9dhcCP4zXvsMS4-gG212LFu79Cpl0MHifKvPk0DTJQja1ulaT4gVuIvmLPihPh1IYMGbEcdX4MFH0Wu_RL6UPINE6esf-oAx8-imKhKITB_R4974rpq9XJk65Kf9R6AJhu072CyOqW1YcmYMkUCqFjdZnRyNgHRT2Q5bMEJ8fv0DwfFV6ynG7n6JGMd_pEnZp0nEvjWXpK6Ft8ZZGOXtFMfmW4vNgFhs6xJ1wnaJWuLXae3V6gTZYxMkeIsyMzlvRSzYBz_rgRBNkvvAwbNvOZ369tKbaElS39hOI1WTaoOsnY2d0Z4mDe4AVbSs3fVJGikzZSa3Ctr1RnqqOztVIRYL1Q7IYRJ02P6egL7sn7RniJ6znNAoPhaWJLYzynWXeQF5YO5U0Zf779qkm3A" />
+                <input type="hidden" name="r"
+                    value="QJznOl.RWpNvkdG1Pf6TAzaNhRIFpH8DJ0w1yAuwRLw-1659202277-0-AcOBapBisncM3qf1RYdkTNlIXCth/TmoAMnk3vJozFlG8/vYeLPpjG389mhQu01aSlpJqFWn0VQf9c/7w3yh85jHmrpaxJtpxTiSL9k+AWm61kE6DkHgJBl5jUc7gu4W3oHdmP4FyOUzhbBpIkOAntSkVJJmgu6SaIE3I9fRAFu7bPxBveT8zZGyVUJSPKpwx/w4rNPzs2VnCeEVL1eOdbLInHYR1kqC8M4JyBynwdVXxIX+j5o/rTrNK8E/W4UZMhuWqIaOnX7FzmceglyBSjDqJFLCt0TOhc66m82Y25Obi8Gvsqn34bjwPA2G8qOvgrHA2RFH6lEQFSdGMzLrF4qU5P9j9FzU1CPSTfGtkKbsMnGcMrtzmyQ7LdMIfghYvnCBXTi82iIzaSwzY3sEnW9KZs24Akxu/AV1E03sqW1CAA1UCRURpX4GKXvD6UYpSgc6++q8naLdRozkLP81T/CvHyIRdQx8vylmVN9u/rPvMbW1jWtniDmuAjBQDUd058YH+IRmm4lREG5JN2yeX083h/BG6tssEQVdTcIgwZRNDB+kK8vtOmywmo5qTAX1VE/sgfPCw5+3Xxu+hhZON3C7VGfrCQI5ZSb6+YvBLXmO26Nlp3fSOyeBwZy3pVuGwv/TrEo+e8USIlIs1T6MQJYQeX/4vOdy89npo6KBqY23giTFDh8EMZo//93hfBsRUbHrY/It6kp42qzsnWTjbkyiqd1zBSpQhuMyuMPeKpQ63oVI2tlGyioLg3HcfhbHQcdpUAWDn8lZ4+GTFVMix+20fGbErkVeBs7WvFSLlZ1YtYpCXrgVaomj7WCr8Icb7ASXKfvEuqC1ZnZgn6Lb6x3dUBGiDtnSFnixHFElIF6nPedVIV0+TxccjlV/LJeyNM58GHtRo4NcmIo1a6kN3vzPAjTUhgDJe4aYP6oVKCRNDcrHlGlLubu6XuIvBFM5Sq401xxahOe3VP2u7JovkzXwfl+yUxQOYaoq1LR+wnDhXgVbBNbM2QfIhez578zu2TN5bu5H14UXZ1E78KA6Op9b/PUgA1AsgTJVRk4M6OQSpa5wRIKkXzxpGIRz6+YBxSjIaX2I220GH4s6Te4CBpq77g6V4CVIkEvqZwbN9hIoAoljWVbEEdb3WmYZqoPxN/8ZIjU7uwQUyDgnCOlc7Z52TgG6nVvj7RVyxv5ugskW+fcOI12o35iYNNpXTh1boHyn7nlPG7wtSsl9UlTss27nd04AIzbH0qyX3kn77yPsobMDYUJ3IGhOujV8Cg08XHFIlYSYGPbqqpog+CuuWtzvwyk5mmHXkJNPyFEZL/irApJbatpGNqgGNnQL+5KYp+/U8/kROLTOWa8tG5609MF+wdrScsfPT9eE+HYh7tEFURnwm8kJtdAadcxYjzO60PFcUI1R5SMGHRflAnpY2gvAzbsSssk1WIF+6eHSe6FLHMXCHMp0w1XkNKpny5Ce3YTKhJ4TRg7HfN1pvet2Duj4G04A328uYUppPlU7Spz0fj5N/FHJf3sPaqJC8jn74L0mT92ecGaxS3ZGvytw51ulA00wgzfZDWL4pirzgYVjUQTqVl9FzWYua4Vk4l3BX0opWKA4FloLTP3ekrvmO/zkztMBV4fvK+F8JIOzLOs4AuoCv8uXl7Ny9wLQI3a0hJAdbXJpI3WV/iuV7da4fQao2Z2HiatQh3ZtdLqWmGtqQlcVtsBrac82eo7mKAfwltTfLlX9Drtp4ohwoFe0Upm+YsfY6DK7zHrk3k9GN7gm6cMi1neNFaqWZR9s8ABDBg==" />
+
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fd3bc9c1d72de');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fd3bc9c1d72de';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script>
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fd3bc9c1d72de</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+</body>
+
+</html>

html_samples/cloudflare_spinner_v1.html

@@ -0,0 +1,167 @@
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '52875',
+                cRay: '732fa2449b567521',
+                cHash: '79cce74ebb92671',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvMTAzLjAuNTA2MC4xMzQgU2FmYXJpLzUzNy4zNg==',
+                    rm: 'R0VU',
+                    d: 'UfK0k9mFeKGEdqoWAUIbk3OXbXe9DOHoYXdKLPyxbICSIQBS4GSNYar0DtbPI7+UQ7UeBZ2XCdQinvgH0pgzJCF1qB0nkXtu0qlLk6EwkrGAKD/pMGFFQF2EaCw3m00/xoRCDgLZRl/wUkRGz3HUOkTuPeKgZjsFyPoPv7MbYSMUtH7QU6ruIh+O3hvDOT2oA/BOKbRMSTnFedTIXADXL6GE8ZyNZ33wJlef5KzT0MHlN+3eZTAt6urCvJaY3MdTXKVye6fwyjqGEksaJ6B85vwrifLTYEU4/bORwXx8mTQTqjo3kh1rATlmthQwBpcQtWXmgDUcJ5gPrOk1fzhqrhO4b++HiIx3P5YZ9Ko2D0NNWeg1AYIwDjh9rZg5m0MmCXh1VqXDbnpseQW1vPkkZAADxyvLf/eEc1o2EpYGpK+qSpMZ4RcngnU0o8A2nS+j/CNsid0315OrYVyOIZcw6L3ovu6yfAAAALyOmg5ctXCqjzRthoibUb58u+myxOtfX1ew9IzNq8Z6t6RlomjR7Iy/7BJiJQNCF98dllNbODHz//TymlI1m8D9w+CYlZFIpiWJVH1M4h+tabH5YrqDVbkJgY6yVAfnr/NI6d6NHrhN+eSW30jkvAmZ6JRMhVWW',
+                    t: 'MTY1OTIwMDI1MS42MjAwMDA=',
+                    m: '/e8nTBb03IHZzN/DSkoHPRu0Ndm3ynYs8g6ZC+VxHcc=',
+                    i1: 'tx+ntPfeE2Gv81s52vIOlA==',
+                    i2: 'fpw8a/EO+Fo2t/ZiNKxEcg==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'Eex9UQDjphKtV6LyVQ95F/MC5kBA3Rj4lC6CudiU3Vs=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521"></script>
+    <script type="text/javascript"
+        src="https://cloudflare.hcaptcha.com/1/api.js?endpoint=https%3A%2F%2Fcloudflare.hcaptcha.com&amp;assethost=https%3A%2F%2Fcf-assets.hcaptcha.com&amp;imghost=https%3A%2F%2Fcf-imgs.hcaptcha.com&amp;render=explicit&amp;recaptchacompat=off&amp;onload=_cf_chl_hload"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer"
+            href="https://chrome.google.com/webstore/detail/privacy-pass/ajhmfdgkijocedmfjonnpjfojldioehi"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: none;"></div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: block; visibility: visible;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fa2449b567521')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" class="fact spacer hidden" style="display: block; visibility: visible;">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">bots
+                    historically made up nearly 40% of all internet traffic?</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: none;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details"> Requests from malicious bots can
+                    pose as legitimate traffic. Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="OghUU_ltYW6I0fpWl7rE4yHBGBPHfpZQIKZRSEpJKjE-1659200251-0-AWB-KR-MabhObmvYa3mR5-xDk3qZVV73547wjnl-QtfPoTxe017AXt4WUskEcVzEIUKC7dsJoiy8ec1NA0fxdnI8X9OfPhtynl00ReWBVZc_3Gba_wigWMmM_9e8PX9vpVDcXpCRbz1BJ5_YLsba9TJM1sp14U9RtIce-tRBB53qoxLxJRz9QFmckEVBvsba4RfoycOvYPMMsfAqSkq13qtsA3Kd6RDB5Rb5-qF8674DsB4AMvd9xu_fBplQqKjOpEtrThCUtw8M2DHY8FUr_owUo1NIS1s6fSBEyHh6ehz9CidJ7zpRwYZFwgz_Pq9i8LmQG_AajozOJJhLp-tox0dptbUZnRNGt3hGQgrNu3jlCfwPC2XVp7xgLvmZoPYrzzrZoi_wErnIvVgyGCw9-sDPblPdvLBUz6uXNreWwThEW6PeRtMXnePO9UwcZmj_2awhwcVSHSLz1t1z22LtVsQ8xNpMbiE7xDvI2D5LNHAPIUC7Wp4AcehWD-fEm0w5jnVTWOmFlVRxtcnYZSMfDSaRUxsZ3hg5B1-ghVMEX6M-r_hAd6pLKNmjIfdl_Nvdm6veQvV-gTFaULbfuhmQQjYEb9G2IptDiNTZs5S7FtmjqVBAA7PmvwBTQwxw86J0cV3v_4pT1Oj8tigwiPny35HMTrKRmRZWaAZudCmWxDZkJIW8Eir7KQ57ba-u9cHh0A">
+                <input type="hidden" name="r"
+                    value="q.UUtPBFcFi4IkcVw3l4U_xJJKDIbHJj7xmuB43IIAI-1659200251-0-Aa4lU5RipD+d4of3hcdQ0rVmZ4ulb3siZYKwhm1jGNiA+/9b8IW1HL8k1GrsYEVexDW7ycP5UINQZ1sYJvZBTCQe3lhyGLHdLZ7KdI9RXKEbPx1NUOR/HthCD0Wbo7H41jbAf7l+HhH0zTLjm77/6NpJZHcgfsbBwwubl4R3oLarzPSByV2PVBnkuMyKCYgibriuMUt2iJHoMLx7Cr+Bmjx1KEFCrPYP0t7vgQs2APTylhL7ebP77XB9ndxU6Of3r4eHnTwLIcomFJ3+jqL6pzFaNoXdUBHrv9oZs/33KZjf2NB8cu5KUpAdM2lp3t5oTSQE19fJVroxmf91hcTdele3F2DAeawFGDwncm/Jo725SlyNk4TqsmR+il7DLkS/FTcCNzQe4cQM6DRWdmF9I1OohAl1/uGXYqUJSK1F45n3gec/pyPTQyZI0OLc7sCGYXfn3VPFsGATkg5mxE9rgZIB2b6ID9JggzIlDdYxlQRWecpruu07KOgk3m7g95lyHNZTohqemo4T8Z2MOZECjmXGMAuwvvk4d5sakVHr39kmAY6aSfXrRB+iONCOKkahbumrVmjLsnMvrpTb0DFE5pRAxwANPZKzb6Ikmlvxh7oJIPOB0mG9hDeoc/AJVlvZJV4CrpDLulNjHetAWXMwMptZuYJGEhcDXxmYj0ybntTCU4Y3JJQc5K+7ehSdnluTvMueWfs628854r4PcOONZzsO337j+3lUxrP5vDUCzYD25FNxvs8jGfqRivqHMOq2z9iOs0sHQTlHroLLSt2G7M50yRJBGTfxrIsvLq+ML3e/mRIkYIQxOcp8ugoPoT4c9gex3OyY0cnnA2/9OibQs9kevwf9DSnutMRRcbIXZI0XO6FY07+MykWqUcXygwMHs1vQxhaQ26NFYwolEWfOL7EQpp4GKyN30nL4nPNil/7GsXIr5SC+o55KI0l3AOEYE1jirVx2G0U7Br7SW80Ih4Fn5U/+4qFfW57GAJrpuk9qjFfJehe7wFBu5bHghEGRhKAu0wvpY7UTc9AiacMfP7ujVWi4DIbTCfOzOgVT8E0T6KaUurBppPJflLQE41c8n29ULyKmki9t8lIKvxYmv/3/AauhXFAExh+JrdnaeSDxhJYjWEUDJiaNvnkDCHMxPs/bePhSg4DYRMh4ngcOHCRkkRlDjipUUgeCrwNBY0qu2DIqLZXI1ZMwU+R0nuWnwc5xJuMHtrLkWbziP0FQcGaF0B6SaFIcOLnWG7YjJZFzxjFpvLb8GnZxk7i2YHCDTn0Stq3JDZHCkjJQjaPMmuK+5KYzfaSHcKOcaQbkbyDjn3t/XQX3a7lknngVchIJVsVn8osqgKvOx3aAdCicYKR6QaukrXHhR9uIEbPdoBYqZPKFz0uvVOShsUx2f65CaI8wWMjOBRWxTK1xUPNsetOiyYSvNwjeULaCXPKLi2qv/cZRRbsr3g5ghdHvNTpD/O0/xUgiziev3/9CpNopyr6VzLar9dJ/s++imXY1w1TCRJ2uCI2H70XGBGWxSZdbnfxU+j3zNCL0dBuabwhDd4ZnOZmlFZjGBiOUpsWdRrHd3c+QpwXdxB3QurRwX6J+LhmkqcWsPhP7LlMnN7dr2HUFZ5FS4LASl5AOf8hjCnO06FT8fWLl1eKVVjCugx9w54qjGqOV8A0v/PdWr7Ic0WfriyYbmwn/XnH8t0ri3bqDZsDkfhQMMF9JSWHEdoGD60a7McGDxr4g9s3LZhq5KozgSvyG+RUBPla8g2zB253hR7amWE5WO4IChl7AXmRB89F9u2+AoDIbefseb3pwG7GfkYpSBwmgJ4Ju4LAWoSfBhZSPMQadHZOCg36R11KesUy+NAy9bvD1bE3UMx9e2NbFohu6sXlilpnxINHp0sFEeulreEjWSQreri1eZeKxV2QfKIzWUiMoNdyT0JzM+/brYzddBpO2DrlnK5bEPWgtu0D7d4Kfm+0T7S//Fq+hxf40lSMPP8cBlan6sEd2iWmZ6gW3z43wNbJaPQIUDgb58ELxaEKQN4tOOy75/XXfISNnhG0K8M79a175WUb8v0A=">
+
+                <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
+                        1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fa2449b567521</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+</body>
+
+</html>

package.json

@@ -0,0 +1,7 @@
+{
+  "name": "flaresolverr",
+  "version": "3.4.6",
+  "description": "Proxy server to bypass Cloudflare protection",
+  "author": "Diego Heras (ngosang / ngosang@hotmail.es)",
+  "license": "MIT"
+}

requirements.txt

@@ -0,0 +1,14 @@
+bottle==0.13.4
+waitress==3.0.2
+selenium==4.39.0
+func-timeout==4.3.5
+prometheus-client==0.23.1
+# Required by undetected_chromedriver
+requests==2.32.5
+certifi==2025.11.12
+websockets==15.0.1
+packaging==25.0
+# Only required for Linux and macOS
+xvfbwrapper==0.2.16; platform_system != "Windows"
+# Only required for Windows
+pefile==2024.8.26; platform_system == "Windows"

src/bottle_plugins/error_plugin.py

@@ -0,0 +1,22 @@
+from bottle import response
+import logging
+
+
+def error_plugin(callback):
+    """
+    Bottle plugin to handle exceptions
+    https://stackoverflow.com/a/32764250
+    """
+
+    def wrapper(*args, **kwargs):
+        try:
+            actual_response = callback(*args, **kwargs)
+        except Exception as e:
+            logging.error(str(e))
+            actual_response = {
+                "error": str(e)
+            }
+            response.status = 500
+        return actual_response
+
+    return wrapper

src/bottle_plugins/logger_plugin.py

@@ -0,0 +1,23 @@
+from bottle import request, response
+import logging
+
+
+def logger_plugin(callback):
+    """
+    Bottle plugin to use logging module
+    https://bottlepy.org/docs/dev/plugindev.html
+
+    Wrap a Bottle request so that a log line is emitted after it's handled.
+    (This decorator can be extended to take the desired logger as a param.)
+    """
+
+    def wrapper(*args, **kwargs):
+        actual_response = callback(*args, **kwargs)
+        if not request.url.endswith("/health"):
+            logging.info('%s %s %s %s' % (request.remote_addr,
+                                          request.method,
+                                          request.url,
+                                          response.status))
+        return actual_response
+
+    return wrapper

src/bottle_plugins/prometheus_plugin.py

@@ -0,0 +1,66 @@
+import logging
+import os
+import urllib.parse
+
+from bottle import request
+from dtos import V1RequestBase, V1ResponseBase
+from metrics import start_metrics_http_server, REQUEST_COUNTER, REQUEST_DURATION
+
+PROMETHEUS_ENABLED = os.environ.get('PROMETHEUS_ENABLED', 'false').lower() == 'true'
+PROMETHEUS_PORT = int(os.environ.get('PROMETHEUS_PORT', 8192))
+
+
+def setup():
+    if PROMETHEUS_ENABLED:
+        start_metrics_http_server(PROMETHEUS_PORT)
+
+
+def prometheus_plugin(callback):
+    """
+    Bottle plugin to expose Prometheus metrics
+    https://bottlepy.org/docs/dev/plugindev.html
+    """
+    def wrapper(*args, **kwargs):
+        actual_response = callback(*args, **kwargs)
+
+        if PROMETHEUS_ENABLED:
+            try:
+                export_metrics(actual_response)
+            except Exception as e:
+                logging.warning("Error exporting metrics: " + str(e))
+
+        return actual_response
+
+    def export_metrics(actual_response):
+        res = V1ResponseBase(actual_response)
+
+        if res.startTimestamp is None or res.endTimestamp is None:
+            # skip management and healthcheck endpoints
+            return
+
+        domain = "unknown"
+        if res.solution and res.solution.url:
+            domain = parse_domain_url(res.solution.url)
+        else:
+            # timeout error
+            req = V1RequestBase(request.json)
+            if req.url:
+                domain = parse_domain_url(req.url)
+
+        run_time = (res.endTimestamp - res.startTimestamp) / 1000
+        REQUEST_DURATION.labels(domain=domain).observe(run_time)
+
+        result = "unknown"
+        if res.message == "Challenge solved!":
+            result = "solved"
+        elif res.message == "Challenge not detected!":
+            result = "not_detected"
+        elif res.message.startswith("Error"):
+            result = "error"
+        REQUEST_COUNTER.labels(domain=domain, result=result).inc()
+
+    def parse_domain_url(url):
+        parsed_url = urllib.parse.urlparse(url)
+        return parsed_url.hostname
+
+    return wrapper

src/build_package.py

@@ -0,0 +1,126 @@
+import os
+import platform
+import shutil
+import subprocess
+import sys
+import zipfile
+import tarfile
+
+import requests
+
+
+def clean_files():
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'build'))
+    except Exception:
+        pass
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist'))
+    except Exception:
+        pass
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome'))
+    except Exception:
+        pass
+
+
+def download_chromium():
+    # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
+    revision = "1522586" if os.name == 'nt' else '1522586'
+    arch = 'Win_x64' if os.name == 'nt' else 'Linux_x64'
+    dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux'
+    dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')
+    dl_path_folder = os.path.join(dl_path, dl_file)
+    dl_path_zip = dl_path_folder + '.zip'
+
+    # response = requests.get(
+    #     f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/LAST_CHANGE',
+    #     timeout=30)
+    # revision = response.text.strip()
+    print("Downloading revision: " + revision)
+
+    os.mkdir(dl_path)
+    with requests.get(
+            f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/{revision}/{dl_file}.zip',
+            stream=True) as r:
+        r.raise_for_status()
+        with open(dl_path_zip, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    print("File downloaded: " + dl_path_zip)
+    with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
+        zip_ref.extractall(dl_path)
+    os.remove(dl_path_zip)
+
+    chrome_path = os.path.join(dl_path, "chrome")
+    shutil.move(dl_path_folder, chrome_path)
+    print("Extracted in: " + chrome_path)
+
+    if os.name != 'nt':
+        # Give executable permissions for *nix
+        # file * | grep executable | cut -d: -f1
+        print("Giving executable permissions...")
+        execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'xdg-mime', 'xdg-settings']
+        for exec_file in execs:
+            exec_path = os.path.join(chrome_path, exec_file)
+            os.chmod(exec_path, 0o755)
+
+
+def run_pyinstaller():
+    sep = ';' if os.name == 'nt' else ':'
+    result = subprocess.run([sys.executable, "-m", "PyInstaller",
+                             "--icon", "resources/flaresolverr_logo.ico",
+                             "--add-data", f"package.json{sep}.",
+                             "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
+                             os.path.join("src", "flaresolverr.py")],
+                            cwd=os.pardir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if result.returncode != 0:
+        print(result.stderr.decode('utf-8'))
+        raise Exception("Error running pyInstaller")
+
+
+def compress_package():
+    dist_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
+    package_folder = os.path.join(dist_folder, 'package')
+    shutil.move(os.path.join(dist_folder, 'flaresolverr'), os.path.join(package_folder, 'flaresolverr'))
+    print("Package folder: " + package_folder)
+
+    compr_format = 'zip' if os.name == 'nt' else 'gztar'
+    compr_file_name = 'flaresolverr_windows_x64' if os.name == 'nt' else 'flaresolverr_linux_x64'
+    compr_file_path = os.path.join(dist_folder, compr_file_name)
+    
+    if compr_format == 'zip':
+        shutil.make_archive(compr_file_path, compr_format, package_folder)
+        print("Compressed file path: " + compr_file_path)
+    else:
+        def _reset_tarinfo(tarinfo):
+            tarinfo.uid = 0
+            tarinfo.gid = 0
+            tarinfo.uname = ""
+            tarinfo.gname = ""
+            return tarinfo
+
+        tar_path = compr_file_path + '.tar.gz'
+        with tarfile.open(tar_path, 'w:gz') as tar:
+            for entry in os.listdir(package_folder):
+                fullpath = os.path.join(package_folder, entry)
+                tar.add(fullpath, arcname=entry, filter=_reset_tarinfo)
+        print("Compressed file path: " + tar_path)
+
+if __name__ == "__main__":
+    print("Building package...")
+    print("Platform: " + platform.platform())
+
+    print("Cleaning previous build...")
+    clean_files()
+
+    print("Downloading Chromium...")
+    download_chromium()
+
+    print("Building pyinstaller executable... ")
+    run_pyinstaller()
+
+    print("Compressing package... ")
+    compress_package()
+
+# NOTE: python -m pip install pyinstaller

src/dtos.py

@@ -0,0 +1,94 @@
+
+STATUS_OK = "ok"
+STATUS_ERROR = "error"
+
+
+class ChallengeResolutionResultT:
+    url: str = None
+    status: int = None
+    headers: list = None
+    response: str = None
+    cookies: list = None
+    userAgent: str = None
+    screenshot: str | None = None
+    turnstile_token: str = None
+
+    def __init__(self, _dict):
+        self.__dict__.update(_dict)
+
+
+class ChallengeResolutionT:
+    status: str = None
+    message: str = None
+    result: ChallengeResolutionResultT = None
+
+    def __init__(self, _dict):
+        self.__dict__.update(_dict)
+        if self.result is not None:
+            self.result = ChallengeResolutionResultT(self.result)
+
+
+class V1RequestBase(object):
+    # V1RequestBase
+    cmd: str = None
+    cookies: list = None
+    maxTimeout: int = None
+    proxy: dict = None
+    session: str = None
+    session_ttl_minutes: int = None
+    headers: list = None  # deprecated v2.0.0, not used
+    userAgent: str = None  # deprecated v2.0.0, not used
+
+    # V1Request
+    url: str = None
+    postData: str = None
+    returnOnlyCookies: bool = None
+    returnScreenshot: bool = None
+    download: bool = None   # deprecated v2.0.0, not used
+    returnRawHtml: bool = None  # deprecated v2.0.0, not used
+    waitInSeconds: int = None
+    # Optional resource blocking flag (blocks images, CSS, and fonts)
+    disableMedia: bool = None
+    # Optional when you've got a turnstile captcha that needs to be clicked after X number of Tab presses
+    tabs_till_verify : int = None
+
+    def __init__(self, _dict):
+        self.__dict__.update(_dict)
+
+
+class V1ResponseBase(object):
+    # V1ResponseBase
+    status: str = None
+    message: str = None
+    session: str = None
+    sessions: list[str] = None
+    startTimestamp: int = None
+    endTimestamp: int = None
+    version: str = None
+
+    # V1ResponseSolution
+    solution: ChallengeResolutionResultT = None
+
+    # hidden vars
+    __error_500__: bool = False
+
+    def __init__(self, _dict):
+        self.__dict__.update(_dict)
+        if self.solution is not None:
+            self.solution = ChallengeResolutionResultT(self.solution)
+
+
+class IndexResponse(object):
+    msg: str = None
+    version: str = None
+    userAgent: str = None
+
+    def __init__(self, _dict):
+        self.__dict__.update(_dict)
+
+
+class HealthResponse(object):
+    status: str = None
+
+    def __init__(self, _dict):
+        self.__dict__.update(_dict)

src/flaresolverr.py

@@ -0,0 +1,152 @@
+import json
+import logging
+import os
+import sys
+
+import certifi
+from bottle import run, response, Bottle, request, ServerAdapter
+
+from bottle_plugins.error_plugin import error_plugin
+from bottle_plugins.logger_plugin import logger_plugin
+from bottle_plugins import prometheus_plugin
+from dtos import V1RequestBase
+import flaresolverr_service
+import utils
+
+env_proxy_url = os.environ.get('PROXY_URL', None)
+env_proxy_username = os.environ.get('PROXY_USERNAME', None)
+env_proxy_password = os.environ.get('PROXY_PASSWORD', None)
+
+
+class JSONErrorBottle(Bottle):
+    """
+    Handle 404 errors
+    """
+    def default_error_handler(self, res):
+        response.content_type = 'application/json'
+        return json.dumps(dict(error=res.body, status_code=res.status_code))
+
+
+app = JSONErrorBottle()
+
+
+@app.route('/')
+def index():
+    """
+    Show welcome message
+    """
+    res = flaresolverr_service.index_endpoint()
+    return utils.object_to_dict(res)
+
+
+@app.route('/health')
+def health():
+    """
+    Healthcheck endpoint.
+    This endpoint is special because it doesn't print traces
+    """
+    res = flaresolverr_service.health_endpoint()
+    return utils.object_to_dict(res)
+
+
+@app.post('/v1')
+def controller_v1():
+    """
+    Controller v1
+    """
+    data = request.json or {}
+    if (('proxy' not in data or not data.get('proxy')) and env_proxy_url is not None and (env_proxy_username is None and env_proxy_password is None)):
+        logging.info('Using proxy URL ENV')
+        data['proxy'] = {"url": env_proxy_url}
+    if (('proxy' not in data or not data.get('proxy')) and env_proxy_url is not None and (env_proxy_username is not None or env_proxy_password is not None)):
+        logging.info('Using proxy URL, username & password ENVs')
+        data['proxy'] = {"url": env_proxy_url, "username": env_proxy_username, "password": env_proxy_password}
+    req = V1RequestBase(data)
+    res = flaresolverr_service.controller_v1_endpoint(req)
+    if res.__error_500__:
+        response.status = 500
+    return utils.object_to_dict(res)
+
+
+if __name__ == "__main__":
+    # check python version
+    if sys.version_info < (3, 9):
+        raise Exception("The Python version is less than 3.9, a version equal to or higher is required.")
+
+    # fix for HEADLESS=false in Windows binary
+    # https://stackoverflow.com/a/27694505
+    if os.name == 'nt':
+        import multiprocessing
+        multiprocessing.freeze_support()
+
+    # fix ssl certificates for compiled binaries
+    # https://github.com/pyinstaller/pyinstaller/issues/7229
+    # https://stackoverflow.com/q/55736855
+    os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
+    os.environ["SSL_CERT_FILE"] = certifi.where()
+
+    # validate configuration
+    log_level = os.environ.get('LOG_LEVEL', 'info').upper()
+    log_file = os.environ.get('LOG_FILE', None)
+    log_html = utils.get_config_log_html()
+    headless = utils.get_config_headless()
+    server_host = os.environ.get('HOST', '0.0.0.0')
+    server_port = int(os.environ.get('PORT', 8191))
+
+    # configure logger
+    logger_format = '%(asctime)s %(levelname)-8s %(message)s'
+    if log_level == 'DEBUG':
+        logger_format = '%(asctime)s %(levelname)-8s ReqId %(thread)s %(message)s'
+    if log_file:
+        log_file = os.path.realpath(log_file)
+        log_path = os.path.dirname(log_file)
+        os.makedirs(log_path, exist_ok=True)
+        logging.basicConfig(
+            format=logger_format,
+            level=log_level,
+            datefmt='%Y-%m-%d %H:%M:%S',
+            handlers=[
+                logging.StreamHandler(sys.stdout),
+                logging.FileHandler(log_file)
+            ]
+        )
+    else:
+        logging.basicConfig(
+            format=logger_format,
+            level=log_level,
+            datefmt='%Y-%m-%d %H:%M:%S',
+            handlers=[
+                logging.StreamHandler(sys.stdout)
+            ]
+        )
+
+    # disable warning traces from urllib3
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    logging.getLogger('selenium.webdriver.remote.remote_connection').setLevel(logging.WARNING)
+    logging.getLogger('undetected_chromedriver').setLevel(logging.WARNING)
+
+    logging.info(f'FlareSolverr {utils.get_flaresolverr_version()}')
+    logging.debug('Debug log enabled')
+
+    # Get current OS for global variable
+    utils.get_current_platform()
+
+    # test browser installation
+    flaresolverr_service.test_browser_installation()
+
+    # start bootle plugins
+    # plugin order is important
+    app.install(logger_plugin)
+    app.install(error_plugin)
+    prometheus_plugin.setup()
+    app.install(prometheus_plugin.prometheus_plugin)
+
+    # start webserver
+    # default server 'wsgiref' does not support concurrent requests
+    # https://github.com/FlareSolverr/FlareSolverr/issues/680
+    # https://github.com/Pylons/waitress/issues/31
+    class WaitressServerPoll(ServerAdapter):
+        def run(self, handler):
+            from waitress import serve
+            serve(handler, host=self.host, port=self.port, asyncore_use_poll=True)
+    run(app, host=server_host, port=server_port, quiet=True, server=WaitressServerPoll)

src/flaresolverr_service.py

@@ -0,0 +1,519 @@
+import logging
+import platform
+import sys
+import time
+from datetime import timedelta
+from html import escape
+from urllib.parse import unquote, quote
+
+from func_timeout import FunctionTimedOut, func_timeout
+from selenium.common import TimeoutException
+from selenium.webdriver.chrome.webdriver import WebDriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.expected_conditions import (
+    presence_of_element_located, staleness_of, title_is)
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.support.wait import WebDriverWait
+
+import utils
+from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT,
+                  ChallengeResolutionT, HealthResponse, IndexResponse,
+                  V1RequestBase, V1ResponseBase)
+from sessions import SessionsStorage
+
+ACCESS_DENIED_TITLES = [
+    # Cloudflare
+    'Access denied',
+    # Cloudflare http://bitturk.net/ Firefox
+    'Attention Required! | Cloudflare'
+]
+ACCESS_DENIED_SELECTORS = [
+    # Cloudflare
+    'div.cf-error-title span.cf-code-label span',
+    # Cloudflare http://bitturk.net/ Firefox
+    '#cf-error-details div.cf-error-overview h1'
+]
+CHALLENGE_TITLES = [
+    # Cloudflare
+    'Just a moment...',
+    # DDoS-GUARD
+    'DDoS-Guard'
+]
+CHALLENGE_SELECTORS = [
+    # Cloudflare
+    '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', '.lds-ring',
+    # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+    'td.info #js_info',
+    # Fairlane / pararius.com
+    'div.vc div.text-box h2'
+]
+
+TURNSTILE_SELECTORS = [
+    "input[name='cf-turnstile-response']"
+]
+
+SHORT_TIMEOUT = 1
+SESSIONS_STORAGE = SessionsStorage()
+
+
+def test_browser_installation():
+    logging.info("Testing web browser installation...")
+    logging.info("Platform: " + platform.platform())
+
+    chrome_exe_path = utils.get_chrome_exe_path()
+    if chrome_exe_path is None:
+        logging.error("Chrome / Chromium web browser not installed!")
+        sys.exit(1)
+    else:
+        logging.info("Chrome / Chromium path: " + chrome_exe_path)
+
+    chrome_major_version = utils.get_chrome_major_version()
+    if chrome_major_version == '':
+        logging.error("Chrome / Chromium version not detected!")
+        sys.exit(1)
+    else:
+        logging.info("Chrome / Chromium major version: " + chrome_major_version)
+
+    logging.info("Launching web browser...")
+    user_agent = utils.get_user_agent()
+    logging.info("FlareSolverr User-Agent: " + user_agent)
+    logging.info("Test successful!")
+
+
+def index_endpoint() -> IndexResponse:
+    res = IndexResponse({})
+    res.msg = "FlareSolverr is ready!"
+    res.version = utils.get_flaresolverr_version()
+    res.userAgent = utils.get_user_agent()
+    return res
+
+
+def health_endpoint() -> HealthResponse:
+    res = HealthResponse({})
+    res.status = STATUS_OK
+    return res
+
+
+def controller_v1_endpoint(req: V1RequestBase) -> V1ResponseBase:
+    start_ts = int(time.time() * 1000)
+    logging.info(f"Incoming request => POST /v1 body: {utils.object_to_dict(req)}")
+    res: V1ResponseBase
+    try:
+        res = _controller_v1_handler(req)
+    except Exception as e:
+        res = V1ResponseBase({})
+        res.__error_500__ = True
+        res.status = STATUS_ERROR
+        res.message = "Error: " + str(e)
+        logging.error(res.message)
+
+    res.startTimestamp = start_ts
+    res.endTimestamp = int(time.time() * 1000)
+    res.version = utils.get_flaresolverr_version()
+    logging.debug(f"Response => POST /v1 body: {utils.object_to_dict(res)}")
+    logging.info(f"Response in {(res.endTimestamp - res.startTimestamp) / 1000} s")
+    return res
+
+
+def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase:
+    # do some validations
+    if req.cmd is None:
+        raise Exception("Request parameter 'cmd' is mandatory.")
+    if req.headers is not None:
+        logging.warning("Request parameter 'headers' was removed in FlareSolverr v2.")
+    if req.userAgent is not None:
+        logging.warning("Request parameter 'userAgent' was removed in FlareSolverr v2.")
+
+    # set default values
+    if req.maxTimeout is None or int(req.maxTimeout) < 1:
+        req.maxTimeout = 60000
+
+    # execute the command
+    res: V1ResponseBase
+    if req.cmd == 'sessions.create':
+        res = _cmd_sessions_create(req)
+    elif req.cmd == 'sessions.list':
+        res = _cmd_sessions_list(req)
+    elif req.cmd == 'sessions.destroy':
+        res = _cmd_sessions_destroy(req)
+    elif req.cmd == 'request.get':
+        res = _cmd_request_get(req)
+    elif req.cmd == 'request.post':
+        res = _cmd_request_post(req)
+    else:
+        raise Exception(f"Request parameter 'cmd' = '{req.cmd}' is invalid.")
+
+    return res
+
+
+def _cmd_request_get(req: V1RequestBase) -> V1ResponseBase:
+    # do some validations
+    if req.url is None:
+        raise Exception("Request parameter 'url' is mandatory in 'request.get' command.")
+    if req.postData is not None:
+        raise Exception("Cannot use 'postBody' when sending a GET request.")
+    if req.returnRawHtml is not None:
+        logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
+    if req.download is not None:
+        logging.warning("Request parameter 'download' was removed in FlareSolverr v2.")
+
+    challenge_res = _resolve_challenge(req, 'GET')
+    res = V1ResponseBase({})
+    res.status = challenge_res.status
+    res.message = challenge_res.message
+    res.solution = challenge_res.result
+    return res
+
+
+def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
+    # do some validations
+    if req.postData is None:
+        raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.")
+    if req.returnRawHtml is not None:
+        logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
+    if req.download is not None:
+        logging.warning("Request parameter 'download' was removed in FlareSolverr v2.")
+
+    challenge_res = _resolve_challenge(req, 'POST')
+    res = V1ResponseBase({})
+    res.status = challenge_res.status
+    res.message = challenge_res.message
+    res.solution = challenge_res.result
+    return res
+
+
+def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
+    logging.debug("Creating new session...")
+
+    session, fresh = SESSIONS_STORAGE.create(session_id=req.session, proxy=req.proxy)
+    session_id = session.session_id
+
+    if not fresh:
+        return V1ResponseBase({
+            "status": STATUS_OK,
+            "message": "Session already exists.",
+            "session": session_id
+        })
+
+    return V1ResponseBase({
+        "status": STATUS_OK,
+        "message": "Session created successfully.",
+        "session": session_id
+    })
+
+
+def _cmd_sessions_list(req: V1RequestBase) -> V1ResponseBase:
+    session_ids = SESSIONS_STORAGE.session_ids()
+
+    return V1ResponseBase({
+        "status": STATUS_OK,
+        "message": "",
+        "sessions": session_ids
+    })
+
+
+def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase:
+    session_id = req.session
+    existed = SESSIONS_STORAGE.destroy(session_id)
+
+    if not existed:
+        raise Exception("The session doesn't exist.")
+
+    return V1ResponseBase({
+        "status": STATUS_OK,
+        "message": "The session has been removed."
+    })
+
+
+def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
+    timeout = int(req.maxTimeout) / 1000
+    driver = None
+    try:
+        if req.session:
+            session_id = req.session
+            ttl = timedelta(minutes=req.session_ttl_minutes) if req.session_ttl_minutes else None
+            session, fresh = SESSIONS_STORAGE.get(session_id, ttl)
+
+            if fresh:
+                logging.debug(f"new session created to perform the request (session_id={session_id})")
+            else:
+                logging.debug(f"existing session is used to perform the request (session_id={session_id}, "
+                              f"lifetime={str(session.lifetime())}, ttl={str(ttl)})")
+
+            driver = session.driver
+        else:
+            driver = utils.get_webdriver(req.proxy)
+            logging.debug('New instance of webdriver has been created to perform the request')
+        return func_timeout(timeout, _evil_logic, (req, driver, method))
+    except FunctionTimedOut:
+        raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
+    except Exception as e:
+        raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
+    finally:
+        if not req.session and driver is not None:
+            if utils.PLATFORM_VERSION == "nt":
+                driver.close()
+            driver.quit()
+            logging.debug('A used instance of webdriver has been destroyed')
+
+
+def click_verify(driver: WebDriver, num_tabs: int = 1):
+    try:
+        logging.debug("Try to find the Cloudflare verify checkbox...")
+        actions = ActionChains(driver)
+        actions.pause(5)
+        for _ in range(num_tabs):
+            actions.send_keys(Keys.TAB).pause(0.1)
+        actions.pause(1)
+        actions.send_keys(Keys.SPACE).perform()
+        
+        logging.debug(f"Cloudflare verify checkbox clicked after {num_tabs} tabs!")
+    except Exception:
+        logging.debug("Cloudflare verify checkbox not found on the page.")
+    finally:
+        driver.switch_to.default_content()
+
+    try:
+        logging.debug("Try to find the Cloudflare 'Verify you are human' button...")
+        button = driver.find_element(
+            by=By.XPATH,
+            value="//input[@type='button' and @value='Verify you are human']",
+        )
+        if button:
+            actions = ActionChains(driver)
+            actions.move_to_element_with_offset(button, 5, 7)
+            actions.click(button)
+            actions.perform()
+            logging.debug("The Cloudflare 'Verify you are human' button found and clicked!")
+    except Exception:
+        logging.debug("The Cloudflare 'Verify you are human' button not found on the page.")
+
+    time.sleep(2)
+
+def _get_turnstile_token(driver: WebDriver, tabs: int):
+    token_input = driver.find_element(By.CSS_SELECTOR, "input[name='cf-turnstile-response']")
+    current_value = token_input.get_attribute("value")
+    while True:
+        click_verify(driver, num_tabs=tabs)
+        turnstile_token = token_input.get_attribute("value")
+        if turnstile_token:
+            if turnstile_token != current_value:
+                logging.info(f"Turnstile token: {turnstile_token}")
+                return turnstile_token
+        logging.debug(f"Failed to extract token possibly click failed")        
+
+        # reset focus
+        driver.execute_script("""
+            let el = document.createElement('button');
+            el.style.position='fixed';
+            el.style.top='0';
+            el.style.left='0';
+            document.body.prepend(el);
+            el.focus();
+        """)
+        time.sleep(1)
+
+def _resolve_turnstile_captcha(req: V1RequestBase, driver: WebDriver):
+    turnstile_token = None
+    if req.tabs_till_verify is not None:
+        logging.debug(f'Navigating to... {req.url} in order to pass the turnstile challenge')
+        driver.get(req.url)
+
+        turnstile_challenge_found = False
+        for selector in TURNSTILE_SELECTORS:
+            found_elements = driver.find_elements(By.CSS_SELECTOR, selector)   
+            if len(found_elements) > 0:
+                turnstile_challenge_found = True
+                logging.info("Turnstile challenge detected. Selector found: " + selector)
+                break
+        if turnstile_challenge_found:
+            turnstile_token = _get_turnstile_token(driver=driver, tabs=req.tabs_till_verify)
+        else:
+            logging.debug(f'Turnstile challenge not found')
+    return turnstile_token
+
+def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT:
+    res = ChallengeResolutionT({})
+    res.status = STATUS_OK
+    res.message = ""
+
+    # optionally block resources like images/css/fonts using CDP
+    disable_media = utils.get_config_disable_media()
+    if req.disableMedia is not None:
+        disable_media = req.disableMedia
+    if disable_media:
+        block_urls = [
+            # Images
+            "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.bmp", "*.svg", "*.ico",
+            "*.PNG", "*.JPG", "*.JPEG", "*.GIF", "*.WEBP", "*.BMP", "*.SVG", "*.ICO",
+            "*.tiff", "*.tif", "*.jpe", "*.apng", "*.avif", "*.heic", "*.heif",
+            "*.TIFF", "*.TIF", "*.JPE", "*.APNG", "*.AVIF", "*.HEIC", "*.HEIF",
+            # Stylesheets
+            "*.css",
+            "*.CSS",
+            # Fonts
+            "*.woff", "*.woff2", "*.ttf", "*.otf", "*.eot",
+            "*.WOFF", "*.WOFF2", "*.TTF", "*.OTF", "*.EOT"
+        ]
+        try:
+            logging.debug("Network.setBlockedURLs: %s", block_urls)
+            driver.execute_cdp_cmd("Network.enable", {})
+            driver.execute_cdp_cmd("Network.setBlockedURLs", {"urls": block_urls})
+        except Exception:
+            # if CDP commands are not available or fail, ignore and continue
+            logging.debug("Network.setBlockedURLs failed or unsupported on this webdriver")
+
+    # navigate to the page
+    logging.debug(f"Navigating to... {req.url}")
+    turnstile_token = None
+
+    if method == "POST":
+        _post_request(req, driver)
+    else:
+        if req.tabs_till_verify is None:
+            driver.get(req.url)
+        else:
+            turnstile_token = _resolve_turnstile_captcha(req, driver)
+
+    # set cookies if required
+    if req.cookies is not None and len(req.cookies) > 0:
+        logging.debug(f'Setting cookies...')
+        for cookie in req.cookies:
+            driver.delete_cookie(cookie['name'])
+            driver.add_cookie(cookie)
+        # reload the page
+        if method == 'POST':
+            _post_request(req, driver)
+        else:
+            driver.get(req.url)
+
+    # wait for the page
+    if utils.get_config_log_html():
+        logging.debug(f"Response HTML:\n{driver.page_source}")
+    html_element = driver.find_element(By.TAG_NAME, "html")
+    page_title = driver.title
+
+    # find access denied titles
+    for title in ACCESS_DENIED_TITLES:
+        if page_title.startswith(title):
+            raise Exception('Cloudflare has blocked this request. '
+                            'Probably your IP is banned for this site, check in your web browser.')
+    # find access denied selectors
+    for selector in ACCESS_DENIED_SELECTORS:
+        found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
+        if len(found_elements) > 0:
+            raise Exception('Cloudflare has blocked this request. '
+                            'Probably your IP is banned for this site, check in your web browser.')
+
+    # find challenge by title
+    challenge_found = False
+    for title in CHALLENGE_TITLES:
+        if title.lower() == page_title.lower():
+            challenge_found = True
+            logging.info("Challenge detected. Title found: " + page_title)
+            break
+    if not challenge_found:
+        # find challenge by selectors
+        for selector in CHALLENGE_SELECTORS:
+            found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
+            if len(found_elements) > 0:
+                challenge_found = True
+                logging.info("Challenge detected. Selector found: " + selector)
+                break
+
+    attempt = 0
+    if challenge_found:
+        while True:
+            try:
+                attempt = attempt + 1
+                # wait until the title changes
+                for title in CHALLENGE_TITLES:
+                    logging.debug("Waiting for title (attempt " + str(attempt) + "): " + title)
+                    WebDriverWait(driver, SHORT_TIMEOUT).until_not(title_is(title))
+
+                # then wait until all the selectors disappear
+                for selector in CHALLENGE_SELECTORS:
+                    logging.debug("Waiting for selector (attempt " + str(attempt) + "): " + selector)
+                    WebDriverWait(driver, SHORT_TIMEOUT).until_not(
+                        presence_of_element_located((By.CSS_SELECTOR, selector)))
+
+                # all elements not found
+                break
+
+            except TimeoutException:
+                logging.debug("Timeout waiting for selector")
+
+                click_verify(driver)
+
+                # update the html (cloudflare reloads the page every 5 s)
+                html_element = driver.find_element(By.TAG_NAME, "html")
+
+        # waits until cloudflare redirection ends
+        logging.debug("Waiting for redirect")
+        # noinspection PyBroadException
+        try:
+            WebDriverWait(driver, SHORT_TIMEOUT).until(staleness_of(html_element))
+        except Exception:
+            logging.debug("Timeout waiting for redirect")
+
+        logging.info("Challenge solved!")
+        res.message = "Challenge solved!"
+    else:
+        logging.info("Challenge not detected!")
+        res.message = "Challenge not detected!"
+
+    challenge_res = ChallengeResolutionResultT({})
+    challenge_res.url = driver.current_url
+    challenge_res.status = 200  # todo: fix, selenium not provides this info
+    challenge_res.cookies = driver.get_cookies()
+    challenge_res.userAgent = utils.get_user_agent(driver)
+    challenge_res.turnstile_token = turnstile_token
+
+    if not req.returnOnlyCookies:
+        challenge_res.headers = {}  # todo: fix, selenium not provides this info
+
+        if req.waitInSeconds and req.waitInSeconds > 0:
+            logging.info("Waiting " + str(req.waitInSeconds) + " seconds before returning the response...")
+            time.sleep(req.waitInSeconds)
+
+        challenge_res.response = driver.page_source
+
+    if req.returnScreenshot:
+        challenge_res.screenshot = driver.get_screenshot_as_base64()
+
+    res.result = challenge_res
+    return res
+
+
+def _post_request(req: V1RequestBase, driver: WebDriver):
+    post_form = f'<form id="hackForm" action="{req.url}" method="POST">'
+    query_string = req.postData if req.postData and req.postData[0] != '?' else req.postData[1:] if req.postData else ''
+    pairs = query_string.split('&')
+    for pair in pairs:
+        parts = pair.split('=', 1)
+        # noinspection PyBroadException
+        try:
+            name = unquote(parts[0])
+        except Exception:
+            name = parts[0]
+        if name == 'submit':
+            continue
+        # noinspection PyBroadException
+        try:
+            value = unquote(parts[1]) if len(parts) > 1 else ''
+        except Exception:
+            value = parts[1] if len(parts) > 1 else ''
+        # Protection of " character, for syntax
+        value=value.replace('"','&quot;')
+        post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
+    post_form += '</form>'
+    html_content = f"""
+        <!DOCTYPE html>
+        <html>
+        <body>
+            {post_form}
+            <script>document.getElementById('hackForm').submit();</script>
+        </body>
+        </html>"""
+    driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))

src/metrics.py

@@ -0,0 +1,32 @@
+import logging
+
+from prometheus_client import Counter, Histogram, start_http_server
+import time
+
+REQUEST_COUNTER = Counter(
+    name='flaresolverr_request',
+    documentation='Total requests with result',
+    labelnames=['domain', 'result']
+)
+REQUEST_DURATION = Histogram(
+    name='flaresolverr_request_duration',
+    documentation='Request duration in seconds',
+    labelnames=['domain'],
+    buckets=[0, 10, 25, 50]
+)
+
+
+def serve(port):
+    start_http_server(port=port)
+    while True:
+        time.sleep(600)
+
+
+def start_metrics_http_server(prometheus_port: int):
+    logging.info(f"Serving Prometheus exporter on http://0.0.0.0:{prometheus_port}/metrics")
+    from threading import Thread
+    Thread(
+        target=serve,
+        kwargs=dict(port=prometheus_port),
+        daemon=True,
+    ).start()

src/sessions.py

@@ -0,0 +1,84 @@
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Optional, Tuple
+from uuid import uuid1
+
+from selenium.webdriver.chrome.webdriver import WebDriver
+
+import utils
+
+
+@dataclass
+class Session:
+    session_id: str
+    driver: WebDriver
+    created_at: datetime
+
+    def lifetime(self) -> timedelta:
+        return datetime.now() - self.created_at
+
+
+class SessionsStorage:
+    """SessionsStorage creates, stores and process all the sessions"""
+
+    def __init__(self):
+        self.sessions = {}
+
+    def create(self, session_id: Optional[str] = None, proxy: Optional[dict] = None,
+               force_new: Optional[bool] = False) -> Tuple[Session, bool]:
+        """create creates new instance of WebDriver if necessary,
+        assign defined (or newly generated) session_id to the instance
+        and returns the session object. If a new session has been created
+        second argument is set to True.
+
+        Note: The function is idempotent, so in case if session_id
+        already exists in the storage a new instance of WebDriver won't be created
+        and existing session will be returned. Second argument defines if 
+        new session has been created (True) or an existing one was used (False).
+        """
+        session_id = session_id or str(uuid1())
+
+        if force_new:
+            self.destroy(session_id)
+
+        if self.exists(session_id):
+            return self.sessions[session_id], False
+
+        driver = utils.get_webdriver(proxy)
+        created_at = datetime.now()
+        session = Session(session_id, driver, created_at)
+
+        self.sessions[session_id] = session
+
+        return session, True
+
+    def exists(self, session_id: str) -> bool:
+        return session_id in self.sessions
+
+    def destroy(self, session_id: str) -> bool:
+        """destroy closes the driver instance and removes session from the storage.
+        The function is noop if session_id doesn't exist.
+        The function returns True if session was found and destroyed,
+        and False if session_id wasn't found.
+        """
+        if not self.exists(session_id):
+            return False
+
+        session = self.sessions.pop(session_id)
+        if utils.PLATFORM_VERSION == "nt":
+            session.driver.close()
+        session.driver.quit()
+        return True
+
+    def get(self, session_id: str, ttl: Optional[timedelta] = None) -> Tuple[Session, bool]:
+        session, fresh = self.create(session_id)
+
+        if ttl is not None and not fresh and session.lifetime() > ttl:
+            logging.debug(f'session\'s lifetime has expired, so the session is recreated (session_id={session_id})')
+            session, fresh = self.create(session_id, force_new=True)
+
+        return session, fresh
+
+    def session_ids(self) -> list[str]:
+        return list(self.sessions.keys())

src/tests.py

@@ -0,0 +1,655 @@
+import unittest
+from typing import Optional
+
+from webtest import TestApp
+
+from dtos import IndexResponse, HealthResponse, V1ResponseBase, STATUS_OK, STATUS_ERROR
+import flaresolverr
+import utils
+
+
+def _find_obj_by_key(key: str, value: str, _list: list) -> Optional[dict]:
+    for obj in _list:
+        if obj[key] == value:
+            return obj
+    return None
+
+
+class TestFlareSolverr(unittest.TestCase):
+
+    proxy_url = "http://127.0.0.1:8888"
+    proxy_socks_url = "socks5://127.0.0.1:1080"
+    google_url = "https://www.google.com"
+    post_url = "https://httpbin.org/post"
+    cloudflare_url = "https://nowsecure.nl/"
+    cloudflare_url_2 = "https://idope.se/torrent-list/harry/"
+    ddos_guard_url = "https://www.litres.ru/"
+    fairlane_url = "https://www.pararius.com/apartments/amsterdam"
+    custom_cloudflare_url = "https://www.muziekfabriek.org/"
+    cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
+
+    app = TestApp(flaresolverr.app)
+    # wait until the server is ready
+    app.get('/')
+
+    def test_wrong_endpoint(self):
+        res = self.app.get('/wrong', status=404)
+        self.assertEqual(res.status_code, 404)
+
+        body = res.json
+        self.assertEqual("Not found: '/wrong'", body['error'])
+        self.assertEqual(404, body['status_code'])
+
+    def test_index_endpoint(self):
+        res = self.app.get('/')
+        self.assertEqual(res.status_code, 200)
+
+        body = IndexResponse(res.json)
+        self.assertEqual("FlareSolverr is ready!", body.msg)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+        self.assertIn("Chrome/", body.userAgent)
+
+    def test_health_endpoint(self):
+        res = self.app.get('/health')
+        self.assertEqual(res.status_code, 200)
+
+        body = HealthResponse(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+
+    def test_v1_endpoint_wrong_cmd(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.bad",
+            "url": self.google_url
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertEqual("Error: Request parameter 'cmd' = 'request.bad' is invalid.", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+    def test_v1_endpoint_request_get_no_cloudflare(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_disable_resources(self):
+        res = self.app.post_json("/v1", {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "disableMedia": True
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_cloudflare_js_1(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.cloudflare_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge solved!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.cloudflare_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>nowSecure</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
+        self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
+        self.assertGreater(len(cf_cookie["value"]), 30)
+
+    def test_v1_endpoint_request_get_cloudflare_js_2(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.cloudflare_url_2
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge solved!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.cloudflare_url_2, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>harry - idope torrent search</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
+        self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
+        self.assertGreater(len(cf_cookie["value"]), 30)
+
+    def test_v1_endpoint_request_get_ddos_guard_js(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.ddos_guard_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge solved!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.ddos_guard_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Литрес", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        cf_cookie = _find_obj_by_key("name", "__ddg1_", solution.cookies)
+        self.assertIsNotNone(cf_cookie, "DDOS-Guard cookie not found")
+        self.assertGreater(len(cf_cookie["value"]), 10)
+
+    def test_v1_endpoint_request_get_fairlane_js(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.fairlane_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge solved!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.fairlane_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Rental Apartments Amsterdam</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        cf_cookie = _find_obj_by_key("name", "fl_pass_v2_b", solution.cookies)
+        self.assertIsNotNone(cf_cookie, "Fairlane cookie not found")
+        self.assertGreater(len(cf_cookie["value"]), 50)
+
+    def test_v1_endpoint_request_get_custom_cloudflare_js(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.custom_cloudflare_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge solved!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.custom_cloudflare_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>MuziekFabriek : Aanmelden</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        cf_cookie = _find_obj_by_key("name", "ct_anti_ddos_key", solution.cookies)
+        self.assertIsNotNone(cf_cookie, "Custom Cloudflare cookie not found")
+        self.assertGreater(len(cf_cookie["value"]), 10)
+
+    # todo: test Cmd 'request.get' should return fail with Cloudflare CAPTCHA
+
+    def test_v1_endpoint_request_get_cloudflare_blocked(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.cloudflare_blocked_url
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertEqual("Error: Error solving the challenge. Cloudflare has blocked this request. "
+                         "Probably your IP is banned for this site, check in your web browser.", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+    def test_v1_endpoint_request_get_cookies_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "cookies": [
+                {
+                    "name": "testcookie1",
+                    "value": "testvalue1"
+                },
+                {
+                    "name": "testcookie2",
+                    "value": "testvalue2"
+                }
+            ]
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 1)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        user_cookie1 = _find_obj_by_key("name", "testcookie1", solution.cookies)
+        self.assertIsNotNone(user_cookie1, "User cookie 1 not found")
+        self.assertEqual("testvalue1", user_cookie1["value"])
+
+        user_cookie2 = _find_obj_by_key("name", "testcookie2", solution.cookies)
+        self.assertIsNotNone(user_cookie2, "User cookie 2 not found")
+        self.assertEqual("testvalue2", user_cookie2["value"])
+
+    def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "returnOnlyCookies": True
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIsNone(solution.headers)
+        self.assertIsNone(solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_http_param(self):
+        """
+        To configure TinyProxy in local:
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_http_param_with_credentials(self):
+        """
+        To configure TinyProxy in local:
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+              * add => BasicAuth testuser testpass
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_url,
+                "username": "testuser",
+                "password": "testpass"
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_socks_param(self):
+        """
+        To configure Dante in local:
+           * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
+           * sudo vim /etc/sockd.conf
+           * sudo systemctl restart sockd.service
+           * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_socks_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_wrong_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": "http://127.0.0.1:43210"
+            }
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertIn("Error: Error solving the challenge. Message: unknown error: net::ERR_PROXY_CONNECTION_FAILED",
+                      body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+    def test_v1_endpoint_request_get_fail_timeout(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "maxTimeout": 10
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertEqual("Error: Error solving the challenge. Timeout after 0.01 seconds.", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+    def test_v1_endpoint_request_get_fail_bad_domain(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": "https://www.google.combad"
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertIn("Message: unknown error: net::ERR_NAME_NOT_RESOLVED", body.message)
+
+    def test_v1_endpoint_request_get_deprecated_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "userAgent": "Test User-Agent"  # was removed in v2, not used
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+
+    def test_v1_endpoint_request_post_no_cloudflare(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.post",
+            "url": self.post_url,
+            "postData": "param1=value1&param2=value2"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.post_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn('"form": {\n    "param1": "value1", \n    "param2": "value2"\n  }', solution.response)
+        self.assertEqual(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_post_cloudflare(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.post",
+            "url": self.cloudflare_url,
+            "postData": "param1=value1&param2=value2"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge solved!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.cloudflare_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>405 Not Allowed</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
+        self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
+        self.assertGreater(len(cf_cookie["value"]), 30)
+
+    def test_v1_endpoint_request_post_fail_no_post_data(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.post",
+            "url": self.google_url
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertIn("Request parameter 'postData' is mandatory in 'request.post' command", body.message)
+
+    def test_v1_endpoint_request_post_deprecated_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.post",
+            "url": self.google_url,
+            "postData": "param1=value1&param2=value2",
+            "userAgent": "Test User-Agent"  # was removed in v2, not used
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+
+    def test_v1_endpoint_sessions_create_without_session(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.create"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Session created successfully.", body.message)
+        self.assertIsNotNone(body.session)
+
+    def test_v1_endpoint_sessions_create_with_session(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_create_session"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Session created successfully.", body.message)
+        self.assertEqual(body.session, "test_create_session")
+
+    def test_v1_endpoint_sessions_create_with_proxy(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "proxy": {
+                "url": self.proxy_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Session created successfully.", body.message)
+        self.assertIsNotNone(body.session)
+
+    def test_v1_endpoint_sessions_list(self):
+        self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_list_sessions"
+        })
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.list"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("", body.message)
+        self.assertGreaterEqual(len(body.sessions), 1)
+        self.assertIn("test_list_sessions", body.sessions)
+
+    def test_v1_endpoint_sessions_destroy_existing_session(self):
+        self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_destroy_sessions"
+        })
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.destroy",
+            "session": "test_destroy_sessions"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("The session has been removed.", body.message)
+
+    def test_v1_endpoint_sessions_destroy_non_existing_session(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.destroy",
+            "session": "non_existing_session_name"
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertEqual("Error: The session doesn't exist.", body.message)
+
+    def test_v1_endpoint_request_get_with_session(self):
+        self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_request_sessions"
+        })
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "session": "test_request_sessions",
+            "url": self.google_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/tests_sites.py

@@ -0,0 +1,102 @@
+import unittest
+
+from webtest import TestApp
+
+from dtos import V1ResponseBase, STATUS_OK
+import flaresolverr
+import utils
+
+
+def _find_obj_by_key(key: str, value: str, _list: list) -> dict | None:
+    for obj in _list:
+        if obj[key] == value:
+            return obj
+    return None
+
+
+def asset_cloudflare_solution(self, res, site_url, site_text):
+    self.assertEqual(res.status_code, 200)
+
+    body = V1ResponseBase(res.json)
+    self.assertEqual(STATUS_OK, body.status)
+    self.assertEqual("Challenge solved!", body.message)
+    self.assertGreater(body.startTimestamp, 10000)
+    self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+    self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+    solution = body.solution
+    self.assertIn(site_url, solution.url)
+    self.assertEqual(solution.status, 200)
+    self.assertIs(len(solution.headers), 0)
+    self.assertIn(site_text, solution.response)
+    self.assertGreater(len(solution.cookies), 0)
+    self.assertIn("Chrome/", solution.userAgent)
+
+    cf_cookie = _find_obj_by_key("name", "cf_clearance", solution.cookies)
+    self.assertIsNotNone(cf_cookie, "Cloudflare cookie not found")
+    self.assertGreater(len(cf_cookie["value"]), 30)
+
+
+class TestFlareSolverr(unittest.TestCase):
+    app = TestApp(flaresolverr.app)
+    # wait until the server is ready
+    app.get('/')
+
+    def test_v1_endpoint_request_get_cloudflare(self):
+        sites_get = [
+            ('nowsecure', 'https://nowsecure.nl', '<title>nowSecure</title>'),
+            ('0magnet', 'https://0magnet.com/search?q=2022', 'Torrent Search - ØMagnet'),
+            ('1337x', 'https://1337x.unblockit.cat/cat/Movies/time/desc/1/', ''),
+            ('avistaz', 'https://avistaz.to/api/v1/jackett/torrents?in=1&type=0&search=',
+             '<title>Access denied</title>'),
+            ('badasstorrents', 'https://badasstorrents.com/torrents/search/720p/date/desc',
+             '<title>Latest Torrents - BadassTorrents</title>'),
+            ('bt4g', 'https://bt4g.org/search/2022', '<title>Download 2022 Torrents - BT4G</title>'),
+            ('cinemaz', 'https://cinemaz.to/api/v1/jackett/torrents?in=1&type=0&search=',
+             '<title>Access denied</title>'),
+            ('epublibre', 'https://epublibre.unblockit.cat/catalogo/index/0/nuevo/todos/sin/todos/--/ajax',
+             '<title>epublibre - catálogo</title>'),
+            ('ext', 'https://ext.to/latest/?order=age&sort=desc',
+             '<title>Download Latest Torrents - EXT Torrents</title>'),
+            ('extratorrent', 'https://extratorrent.st/search/?srt=added&order=desc&search=720p&new=1&x=0&y=0',
+             'Page 1 - ExtraTorrent'),
+            ('idope', 'https://idope.se/browse.html', '<title>Recent Torrents</title>'),
+            ('limetorrents', 'https://limetorrents.unblockninja.com/latest100',
+             '<title>Latest 100 torrents - LimeTorrents</title>'),
+            ('privatehd', 'https://privatehd.to/api/v1/jackett/torrents?in=1&type=0&search=',
+             '<title>Access denied</title>'),
+            ('torrentcore', 'https://torrentcore.xyz/index', '<title>Torrent[CORE] - Torrent community.</title>'),
+            ('torrentqq223', 'https://torrentqq223.com/torrent/newest.html', 'https://torrentqq223.com/ads/'),
+            ('36dm', 'https://www.36dm.club/1.html', 'https://www.36dm.club/yesterday-1.html'),
+            ('erai-raws', 'https://www.erai-raws.info/feed/?type=magnet', '403 Forbidden'),
+            ('teamos', 'https://www.teamos.xyz/torrents/?filename=&freeleech=',
+             '<title>Log in | Team OS : Your Only Destination To Custom OS !!</title>'),
+            ('yts', 'https://yts.unblockninja.com/api/v2/list_movies.json?query_term=&limit=50&sort=date_added',
+             '{"movie_count":')
+        ]
+        for site_name, site_url, site_text in sites_get:
+            with self.subTest(msg=site_name):
+                res = self.app.post_json('/v1', {
+                    "cmd": "request.get",
+                    "url": site_url
+                })
+                asset_cloudflare_solution(self, res, site_url, site_text)
+
+    def test_v1_endpoint_request_post_cloudflare(self):
+        sites_post = [
+            ('nnmclub', 'https://nnmclub.to/forum/tracker.php', '<title>Трекер :: NNM-Club</title>',
+             'prev_sd=0&prev_a=0&prev_my=0&prev_n=0&prev_shc=0&prev_shf=1&prev_sha=1&prev_shs=0&prev_shr=0&prev_sht=0&f%5B%5D=-1&o=1&s=2&tm=-1&shf=1&sha=1&ta=-1&sns=-1&sds=-1&nm=&pn=&submit=%CF%EE%E8%F1%EA')
+        ]
+
+        for site_name, site_url, site_text, post_data in sites_post:
+            with self.subTest(msg=site_name):
+                res = self.app.post_json('/v1', {
+                    "cmd": "request.post",
+                    "url": site_url,
+                    "postData": post_data
+                })
+                asset_cloudflare_solution(self, res, site_url, site_text)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/undetected_chromedriver/__init__.py

@@ -0,0 +1,910 @@
+#!/usr/bin/env python3
+
+"""
+
+         888                                                  888         d8b
+         888                                                  888         Y8P
+         888                                                  888
+ .d8888b 88888b.  888d888 .d88b.  88888b.d88b.   .d88b.   .d88888 888d888 888 888  888  .d88b.  888d888
+d88P"    888 "88b 888P"  d88""88b 888 "888 "88b d8P  Y8b d88" 888 888P"   888 888  888 d8P  Y8b 888P"
+888      888  888 888    888  888 888  888  888 88888888 888  888 888     888 Y88  88P 88888888 888
+Y88b.    888  888 888    Y88..88P 888  888  888 Y8b.     Y88b 888 888     888  Y8bd8P  Y8b.     888
+ "Y8888P 888  888 888     "Y88P"  888  888  888  "Y8888   "Y88888 888     888   Y88P    "Y8888  888   88888888
+
+by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
+
+"""
+from __future__ import annotations
+
+
+__version__ = "3.5.5"
+
+import json
+import logging
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from weakref import finalize
+
+import selenium.webdriver.chrome.service
+import selenium.webdriver.chrome.webdriver
+from selenium.webdriver.common.by import By
+import selenium.webdriver.chromium.service
+import selenium.webdriver.remote.command
+import selenium.webdriver.remote.webdriver
+
+from .cdp import CDP
+from .dprocess import start_detached
+from .options import ChromeOptions
+from .patcher import IS_POSIX
+from .patcher import Patcher
+from .reactor import Reactor
+from .webelement import UCWebElement
+from .webelement import WebElement
+
+
+__all__ = (
+    "Chrome",
+    "ChromeOptions",
+    "Patcher",
+    "Reactor",
+    "CDP",
+    "find_chrome_executable",
+)
+
+logger = logging.getLogger("uc")
+logger.setLevel(logging.getLogger().getEffectiveLevel())
+
+
+class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
+    """
+
+    Controls the ChromeDriver and allows you to drive the browser.
+
+    The webdriver file will be downloaded by this module automatically,
+    you do not need to specify this. however, you may if you wish.
+
+    Attributes
+    ----------
+
+    Methods
+    -------
+
+    reconnect()
+
+        this can be useful in case of heavy detection methods
+        -stops the chromedriver service which runs in the background
+        -starts the chromedriver service which runs in the background
+        -recreate session
+
+
+    start_session(capabilities=None, browser_profile=None)
+
+        differentiates from the regular method in that it does not
+        require a capabilities argument. The capabilities are automatically
+        recreated from the options at creation time.
+
+    --------------------------------------------------------------------------
+        NOTE:
+            Chrome has everything included to work out of the box.
+            it does not `need` customizations.
+            any customizations MAY lead to trigger bot migitation systems.
+
+    --------------------------------------------------------------------------
+    """
+
+    _instances = set()
+    session_id = None
+    debug = False
+
+    def __init__(
+        self,
+        options=None,
+        user_data_dir=None,
+        driver_executable_path=None,
+        browser_executable_path=None,
+        port=0,
+        enable_cdp_events=False,
+        # service_args=None,
+        # service_creationflags=None,
+        desired_capabilities=None,
+        advanced_elements=False,
+        # service_log_path=None,
+        keep_alive=True,
+        log_level=0,
+        headless=False,
+        version_main=None,
+        patcher_force_close=False,
+        suppress_welcome=True,
+        use_subprocess=False,
+        debug=False,
+        no_sandbox=True,
+        windows_headless=False,
+        user_multi_procs: bool = False,
+        **kw,
+    ):
+        """
+        Creates a new instance of the chrome driver.
+
+        Starts the service and then creates new instance of chrome driver.
+
+        Parameters
+        ----------
+
+        options: ChromeOptions, optional, default: None - automatic useful defaults
+            this takes an instance of ChromeOptions, mainly to customize browser behavior.
+            anything other dan the default, for example extensions or startup options
+            are not supported in case of failure, and can probably lowers your undetectability.
+
+
+        user_data_dir: str , optional, default: None (creates temp profile)
+            if user_data_dir is a path to a valid chrome profile directory, use it,
+            and turn off automatic removal mechanism at exit.
+
+        driver_executable_path: str, optional, default: None(=downloads and patches new binary)
+
+        browser_executable_path: str, optional, default: None - use find_chrome_executable
+            Path to the browser executable.
+            If not specified, make sure the executable's folder is in $PATH
+
+        port: int, optional, default: 0
+            port to be used by the chromedriver executable, this is NOT the debugger port.
+            leave it at 0 unless you know what you are doing.
+            the default value of 0 automatically picks an available port.
+
+        enable_cdp_events: bool, default: False
+            :: currently for chrome only
+            this enables the handling of wire messages
+            when enabled, you can subscribe to CDP events by using:
+
+                driver.add_cdp_listener("Network.dataReceived", yourcallback)
+                # yourcallback is an callable which accepts exactly 1 dict as parameter
+
+
+        service_args: list of str, optional, default: None
+            arguments to pass to the driver service
+
+        desired_capabilities: dict, optional, default: None - auto from config
+            Dictionary object with non-browser specific capabilities only, such as "item" or "loggingPref".
+
+        advanced_elements:  bool, optional, default: False
+            makes it easier to recognize elements like you know them from html/browser inspection, especially when working
+            in an interactive environment
+
+            default webelement repr:
+            <selenium.webdriver.remote.webelement.WebElement (session="85ff0f671512fa535630e71ee951b1f2", element="6357cb55-92c3-4c0f-9416-b174f9c1b8c4")>
+
+            advanced webelement repr
+            <WebElement(<a class="mobile-show-inline-block mc-update-infos init-ok" href="#" id="main-cat-switcher-mobile">)>
+
+            note: when retrieving large amounts of elements ( example: find_elements_by_tag("*") ) and print them, it does take a little more time.
+
+
+        service_log_path: str, optional, default: None
+             path to log information from the driver.
+
+        keep_alive: bool, optional, default: True
+             Whether to configure ChromeRemoteConnection to use HTTP keep-alive.
+
+        log_level: int, optional, default: adapts to python global log level
+
+        headless: bool, optional, default: False
+            can also be specified in the options instance.
+            Specify whether you want to use the browser in headless mode.
+            warning: this lowers undetectability and not fully supported.
+
+        version_main: int, optional, default: None (=auto)
+            if you, for god knows whatever reason, use
+            an older version of Chrome. You can specify it's full rounded version number
+            here. Example: 87 for all versions of 87
+
+        patcher_force_close: bool, optional, default: False
+            instructs the patcher to do whatever it can to access the chromedriver binary
+            if the file is locked, it will force shutdown all instances.
+            setting it is not recommended, unless you know the implications and think
+            you might need it.
+
+        suppress_welcome: bool, optional , default: True
+            a "welcome" alert might show up on *nix-like systems asking whether you want to set
+            chrome as your default browser, and if you want to send even more data to google.
+            now, in case you are nag-fetishist, or a diagnostics data feeder to google, you can set this to False.
+            Note: if you don't handle the nag screen in time, the browser loses it's connection and throws an Exception.
+
+        use_subprocess: bool, optional , default: True,
+
+            False (the default) makes sure Chrome will get it's own process (so no subprocess of chromedriver.exe or python
+                This fixes a LOT of issues, like multithreaded run, but mst importantly. shutting corectly after
+                program exits or using .quit()
+                you should be knowing what you're doing, and know how python works.
+
+              unfortunately, there  is always an edge case in which one would like to write an single script with the only contents being:
+              --start script--
+              import undetected_chromedriver as uc
+              d = uc.Chrome()
+              d.get('https://somesite/')
+              ---end script --
+
+              and will be greeted with an error, since the program exists before chrome has a change to launch.
+              in that case you can set this to `True`. The browser will start via subprocess, and will keep running most of times.
+              ! setting it to True comes with NO support when being detected. !
+
+        no_sandbox: bool, optional, default=True
+             uses the --no-sandbox option, and additionally does suppress the "unsecure option" status bar
+             this option has a default of True since many people seem to run this as root (....) , and chrome does not start
+             when running as root without using --no-sandbox flag.
+
+        user_multi_procs:
+            set to true when you are using multithreads/multiprocessing
+            ensures not all processes are trying to modify a binary which is in use by another.
+            for this to work. YOU MUST HAVE AT LEAST 1 UNDETECTED_CHROMEDRIVER BINARY IN YOUR ROAMING DATA FOLDER.
+            this requirement can be easily satisfied, by just running this program "normal" and close/kill it.
+
+
+        """
+
+        finalize(self, self._ensure_close, self)
+        self.debug = debug
+        self.patcher = Patcher(
+            executable_path=driver_executable_path,
+            force=patcher_force_close,
+            version_main=version_main,
+            user_multi_procs=user_multi_procs,
+        )
+        # self.patcher.auto(user_multiprocess = user_multi_num_procs)
+        self.patcher.auto()
+
+        # self.patcher = patcher
+        if not options:
+            options = ChromeOptions()
+
+        try:
+            if hasattr(options, "_session") and options._session is not None:
+                #  prevent reuse of options,
+                #  as it just appends arguments, not replace them
+                #  you'll get conflicts starting chrome
+                raise RuntimeError("you cannot reuse the ChromeOptions object")
+        except AttributeError:
+            pass
+
+        options._session = self
+
+        if not options.debugger_address:
+            debug_port = (
+                port
+                if port != 0
+                else selenium.webdriver.common.service.utils.free_port()
+            )
+            debug_host = "127.0.0.1"
+            options.debugger_address = "%s:%d" % (debug_host, debug_port)
+        else:
+            debug_host, debug_port = options.debugger_address.split(":")
+            debug_port = int(debug_port)
+
+        if enable_cdp_events:
+            options.set_capability(
+                "goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"}
+            )
+
+        options.add_argument("--remote-debugging-host=%s" % debug_host)
+        options.add_argument("--remote-debugging-port=%s" % debug_port)
+
+        if user_data_dir:
+            options.add_argument("--user-data-dir=%s" % user_data_dir)
+
+        language, keep_user_data_dir = None, bool(user_data_dir)
+
+        # see if a custom user profile is specified in options
+        for arg in options.arguments:
+
+            if any([_ in arg for _ in ("--headless", "headless")]):
+                options.arguments.remove(arg)
+                options.headless = True
+
+            if "lang" in arg:
+                m = re.search("(?:--)?lang(?:[ =])?(.*)", arg)
+                try:
+                    language = m[1]
+                except IndexError:
+                    logger.debug("will set the language to en-US,en;q=0.9")
+                    language = "en-US,en;q=0.9"
+
+            if "user-data-dir" in arg:
+                m = re.search("(?:--)?user-data-dir(?:[ =])?(.*)", arg)
+                try:
+                    user_data_dir = m[1]
+                    logger.debug(
+                        "user-data-dir found in user argument %s => %s" % (arg, m[1])
+                    )
+                    keep_user_data_dir = True
+
+                except IndexError:
+                    logger.debug(
+                        "no user data dir could be extracted from supplied argument %s "
+                        % arg
+                    )
+
+        if not user_data_dir:
+            # backward compatiblity
+            # check if an old uc.ChromeOptions is used, and extract the user data dir
+
+            if hasattr(options, "user_data_dir") and getattr(
+                options, "user_data_dir", None
+            ):
+                import warnings
+
+                warnings.warn(
+                    "using ChromeOptions.user_data_dir might stop working in future versions."
+                    "use uc.Chrome(user_data_dir='/xyz/some/data') in case you need existing profile folder"
+                )
+                options.add_argument("--user-data-dir=%s" % options.user_data_dir)
+                keep_user_data_dir = True
+                logger.debug(
+                    "user_data_dir property found in options object: %s" % user_data_dir
+                )
+
+            else:
+                user_data_dir = os.path.normpath(tempfile.mkdtemp())
+                keep_user_data_dir = False
+                arg = "--user-data-dir=%s" % user_data_dir
+                options.add_argument(arg)
+                logger.debug(
+                    "created a temporary folder in which the user-data (profile) will be stored during this\n"
+                    "session, and added it to chrome startup arguments: %s" % arg
+                )
+
+        if not language:
+            try:
+                import locale
+
+                language = locale.getdefaultlocale()[0].replace("_", "-")
+            except Exception:
+                pass
+            if not language:
+                language = "en-US"
+
+        options.add_argument("--lang=%s" % language)
+
+        if not options.binary_location:
+            options.binary_location = (
+                browser_executable_path or find_chrome_executable()
+            )
+
+        if not options.binary_location or not \
+                pathlib.Path(options.binary_location).exists():
+                raise FileNotFoundError(
+                    "\n---------------------\n"
+                    "Could not determine browser executable."
+                    "\n---------------------\n"
+                    "Make sure your browser is installed in the default location (path).\n"
+                    "If you are sure about the browser executable, you can specify it using\n"
+                    "the `browser_executable_path='{}` parameter.\n\n"
+                    .format("/path/to/browser/executable" if IS_POSIX else "c:/path/to/your/browser.exe")
+                )
+
+        self._delay = 3
+
+        self.user_data_dir = user_data_dir
+        self.keep_user_data_dir = keep_user_data_dir
+
+        if suppress_welcome:
+            options.arguments.extend(["--no-default-browser-check", "--no-first-run"])
+        if no_sandbox:
+            options.arguments.extend(["--no-sandbox", "--test-type"])
+
+        if headless or getattr(options, 'headless', None):
+            #workaround until a better checking is found
+            try:
+                v_main = int(self.patcher.version_main) if self.patcher.version_main else 108
+                if v_main < 108:
+                    options.add_argument("--headless=chrome")
+                elif v_main >= 108:
+                    options.add_argument("--headless=new")
+            except:
+                logger.warning("could not detect version_main."
+                               "therefore, we are assuming it is chrome 108 or higher")
+                options.add_argument("--headless=new")
+
+        options.add_argument("--window-size=1920,1080")
+        options.add_argument("--start-maximized")
+        options.add_argument("--no-sandbox")
+        # fixes "could not connect to chrome" error when running
+        # on linux using privileged user like root (which i don't recommend)
+
+        options.add_argument(
+            "--log-level=%d" % log_level
+            or divmod(logging.getLogger().getEffectiveLevel(), 10)[0]
+        )
+
+        if hasattr(options, "handle_prefs"):
+            options.handle_prefs(user_data_dir)
+
+        # fix exit_type flag to prevent tab-restore nag
+        try:
+            with open(
+                os.path.join(user_data_dir, "Default/Preferences"),
+                encoding="latin1",
+                mode="r+",
+            ) as fs:
+                config = json.load(fs)
+                if config["profile"]["exit_type"] is not None:
+                    # fixing the restore-tabs-nag
+                    config["profile"]["exit_type"] = None
+                fs.seek(0, 0)
+                json.dump(config, fs)
+                fs.truncate()  # the file might be shorter
+                logger.debug("fixed exit_type flag")
+        except Exception as e:
+            logger.debug("did not find a bad exit_type flag ")
+
+        self.options = options
+
+        if not desired_capabilities:
+            desired_capabilities = options.to_capabilities()
+
+        if not use_subprocess and not windows_headless:
+            self.browser_pid = start_detached(
+                options.binary_location, *options.arguments
+            )
+        else:
+            startupinfo = None
+            if os.name == 'nt' and windows_headless:
+                # STARTUPINFO() is Windows only
+                startupinfo = subprocess.STARTUPINFO()
+                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            browser = subprocess.Popen(
+                [options.binary_location, *options.arguments],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                close_fds=IS_POSIX,
+                startupinfo=startupinfo
+            )
+            self.browser_pid = browser.pid
+
+
+        service = selenium.webdriver.chromium.service.ChromiumService(
+            self.patcher.executable_path
+        )
+
+        super().__init__(
+            service=service,
+            options=options,
+            keep_alive=keep_alive,
+        )
+
+        self.reactor = None
+
+        if enable_cdp_events:
+            if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
+                logging.getLogger(
+                    "selenium.webdriver.remote.remote_connection"
+                ).setLevel(20)
+            reactor = Reactor(self)
+            reactor.start()
+            self.reactor = reactor
+
+        if advanced_elements:
+            self._web_element_cls = UCWebElement
+        else:
+            self._web_element_cls = WebElement
+
+        if headless or getattr(options, 'headless', None):
+            self._configure_headless()
+
+    def _configure_headless(self):
+        orig_get = self.get
+        logger.info("setting properties for headless")
+
+        def get_wrapped(*args, **kwargs):
+            if self.execute_script("return navigator.webdriver"):
+                logger.info("patch navigator.webdriver")
+                self.execute_cdp_cmd(
+                    "Page.addScriptToEvaluateOnNewDocument",
+                    {
+                        "source": """
+                                Object.defineProperty(window, "navigator", {
+                                  value: new Proxy(navigator, {
+                                    has: (target, key) => (key === "webdriver" ? false : key in target),
+                                    get: (target, key) =>
+                                      key === "webdriver"
+                                        ? false
+                                        : typeof target[key] === "function"
+                                        ? target[key].bind(target)
+                                        : target[key],
+                                  }),
+                                });
+                    """
+                    },
+                )
+
+                logger.info("patch user-agent string")
+                self.execute_cdp_cmd(
+                    "Network.setUserAgentOverride",
+                    {
+                        "userAgent": self.execute_script(
+                            "return navigator.userAgent"
+                        ).replace("Headless", "")
+                    },
+                )
+                self.execute_cdp_cmd(
+                    "Page.addScriptToEvaluateOnNewDocument",
+                    {
+                        "source": """
+                            Object.defineProperty(navigator, 'maxTouchPoints', {get: () => 1});
+                            Object.defineProperty(navigator.connection, 'rtt', {get: () => 100});
+
+                            // https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/chrome-runtime.js
+                            window.chrome = {
+                                app: {
+                                    isInstalled: false,
+                                    InstallState: {
+                                        DISABLED: 'disabled',
+                                        INSTALLED: 'installed',
+                                        NOT_INSTALLED: 'not_installed'
+                                    },
+                                    RunningState: {
+                                        CANNOT_RUN: 'cannot_run',
+                                        READY_TO_RUN: 'ready_to_run',
+                                        RUNNING: 'running'
+                                    }
+                                },
+                                runtime: {
+                                    OnInstalledReason: {
+                                        CHROME_UPDATE: 'chrome_update',
+                                        INSTALL: 'install',
+                                        SHARED_MODULE_UPDATE: 'shared_module_update',
+                                        UPDATE: 'update'
+                                    },
+                                    OnRestartRequiredReason: {
+                                        APP_UPDATE: 'app_update',
+                                        OS_UPDATE: 'os_update',
+                                        PERIODIC: 'periodic'
+                                    },
+                                    PlatformArch: {
+                                        ARM: 'arm',
+                                        ARM64: 'arm64',
+                                        MIPS: 'mips',
+                                        MIPS64: 'mips64',
+                                        X86_32: 'x86-32',
+                                        X86_64: 'x86-64'
+                                    },
+                                    PlatformNaclArch: {
+                                        ARM: 'arm',
+                                        MIPS: 'mips',
+                                        MIPS64: 'mips64',
+                                        X86_32: 'x86-32',
+                                        X86_64: 'x86-64'
+                                    },
+                                    PlatformOs: {
+                                        ANDROID: 'android',
+                                        CROS: 'cros',
+                                        LINUX: 'linux',
+                                        MAC: 'mac',
+                                        OPENBSD: 'openbsd',
+                                        WIN: 'win'
+                                    },
+                                    RequestUpdateCheckStatus: {
+                                        NO_UPDATE: 'no_update',
+                                        THROTTLED: 'throttled',
+                                        UPDATE_AVAILABLE: 'update_available'
+                                    }
+                                }
+                            }
+
+                            // https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-permissions.js
+                            if (!window.Notification) {
+                                window.Notification = {
+                                    permission: 'denied'
+                                }
+                            }
+
+                            const originalQuery = window.navigator.permissions.query
+                            window.navigator.permissions.__proto__.query = parameters =>
+                                parameters.name === 'notifications'
+                                    ? Promise.resolve({ state: window.Notification.permission })
+                                    : originalQuery(parameters)
+
+                            const oldCall = Function.prototype.call
+                            function call() {
+                                return oldCall.apply(this, arguments)
+                            }
+                            Function.prototype.call = call
+
+                            const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString')
+                            const oldToString = Function.prototype.toString
+
+                            function functionToString() {
+                                if (this === window.navigator.permissions.query) {
+                                    return 'function query() { [native code] }'
+                                }
+                                if (this === functionToString) {
+                                    return nativeToStringFunctionString
+                                }
+                                return oldCall.call(oldToString, this)
+                            }
+                            // eslint-disable-next-line
+                            Function.prototype.toString = functionToString
+                            """
+                    },
+                )
+            return orig_get(*args, **kwargs)
+
+        self.get = get_wrapped
+
+    # def _get_cdc_props(self):
+    #     return self.execute_script(
+    #         """
+    #         let objectToInspect = window,
+    #             result = [];
+    #         while(objectToInspect !== null)
+    #         { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
+    #           objectToInspect = Object.getPrototypeOf(objectToInspect); }
+    #
+    #         return result.filter(i => i.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig))
+    #         """
+    #     )
+    #
+    # def _hook_remove_cdc_props(self):
+    #     self.execute_cdp_cmd(
+    #         "Page.addScriptToEvaluateOnNewDocument",
+    #         {
+    #             "source": """
+    #                 let objectToInspect = window,
+    #                     result = [];
+    #                 while(objectToInspect !== null)
+    #                 { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
+    #                   objectToInspect = Object.getPrototypeOf(objectToInspect); }
+    #                 result.forEach(p => p.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig)
+    #                                     &&delete window[p]&&console.log('removed',p))
+    #                 """
+    #         },
+    #     )
+
+    def get(self, url):
+        # if self._get_cdc_props():
+        #     self._hook_remove_cdc_props()
+        return super().get(url)
+
+    def add_cdp_listener(self, event_name, callback):
+        if (
+            self.reactor
+            and self.reactor is not None
+            and isinstance(self.reactor, Reactor)
+        ):
+            self.reactor.add_event_handler(event_name, callback)
+            return self.reactor.handlers
+        return False
+
+    def clear_cdp_listeners(self):
+        if self.reactor and isinstance(self.reactor, Reactor):
+            self.reactor.handlers.clear()
+
+    def window_new(self):
+        self.execute(
+            selenium.webdriver.remote.command.Command.NEW_WINDOW, {"type": "window"}
+        )
+
+    def tab_new(self, url: str):
+        """
+        this opens a url in a new tab.
+        apparently, that passes all tests directly!
+
+        Parameters
+        ----------
+        url
+
+        Returns
+        -------
+
+        """
+        if not hasattr(self, "cdp"):
+            from .cdp import CDP
+
+            cdp = CDP(self.options)
+            cdp.tab_new(url)
+
+    def reconnect(self, timeout=0.1):
+        try:
+            self.service.stop()
+        except Exception as e:
+            logger.debug(e)
+        time.sleep(timeout)
+        try:
+            self.service.start()
+        except Exception as e:
+            logger.debug(e)
+
+        try:
+            self.start_session()
+        except Exception as e:
+            logger.debug(e)
+
+    def start_session(self, capabilities=None, browser_profile=None):
+        if not capabilities:
+            capabilities = self.options.to_capabilities()
+        super().start_session(capabilities)
+        # super(Chrome, self).start_session(capabilities, browser_profile) # Original explicit call commented out
+
+    def find_elements_recursive(self, by, value):
+        """
+        find elements in all frames
+        this is a generator function, which is needed
+            since if it would return a list of elements, they
+            will be stale on arrival.
+        using generator, when the element is returned we are in the correct frame
+        to use it directly
+        Args:
+            by: By
+            value: str
+        Returns: Generator[webelement.WebElement]
+        """
+        def search_frame(f=None):
+            if not f:
+                # ensure we are on main content frame
+                self.switch_to.default_content()
+            else:
+                self.switch_to.frame(f)
+            for elem in self.find_elements(by, value):
+                yield elem
+            # switch back to main content, otherwise we will get StaleElementReferenceException
+            self.switch_to.default_content()
+
+        # search root frame
+        for elem in search_frame():
+            yield elem
+        # get iframes
+        frames = self.find_elements('css selector', 'iframe')
+
+        # search per frame
+        for f in frames:
+            for elem in search_frame(f):
+                yield elem
+
+    def quit(self):
+        try:
+            self.service.stop()
+            self.service.process.kill()
+            self.command_executor.close()
+            self.service.process.wait(5)
+            logger.debug("webdriver process ended")
+        except (AttributeError, RuntimeError, OSError):
+            pass
+        try:
+            self.reactor.event.set()
+            logger.debug("shutting down reactor")
+        except AttributeError:
+            pass
+        try:
+            os.kill(self.browser_pid, 15)
+            logger.debug("gracefully closed browser")
+        except Exception as e:  # noqa
+            pass
+        if (
+            hasattr(self, "keep_user_data_dir")
+            and hasattr(self, "user_data_dir")
+            and not self.keep_user_data_dir
+        ):
+            for _ in range(5):
+                try:
+                    shutil.rmtree(self.user_data_dir, ignore_errors=False)
+                except FileNotFoundError:
+                    pass
+                except (RuntimeError, OSError, PermissionError) as e:
+                    logger.debug(
+                        "When removing the temp profile, a %s occured: %s\nretrying..."
+                        % (e.__class__.__name__, e)
+                    )
+                else:
+                    logger.debug("successfully removed %s" % self.user_data_dir)
+                    break
+
+                try:
+                    time.sleep(0.1)
+                except OSError:
+                    pass
+
+        # dereference patcher, so patcher can start cleaning up as well.
+        # this must come last, otherwise it will throw 'in use' errors
+        self.patcher = None
+
+    def __getattribute__(self, item):
+        if not super().__getattribute__("debug"):
+            return super().__getattribute__(item)
+        else:
+            import inspect
+
+            original = super().__getattribute__(item)
+            if inspect.ismethod(original) and not inspect.isclass(original):
+
+                def newfunc(*args, **kwargs):
+                    logger.debug(
+                        "calling %s with args %s and kwargs %s\n"
+                        % (original.__qualname__, args, kwargs)
+                    )
+                    return original(*args, **kwargs)
+
+                return newfunc
+            return original
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.service.stop()
+        time.sleep(self._delay)
+        self.service.start()
+        self.start_session()
+
+    def __hash__(self):
+        return hash(self.options.debugger_address)
+
+    def __dir__(self):
+        return object.__dir__(self)
+
+    def __del__(self):
+        try:
+            self.service.process.kill()
+        except:  # noqa
+            pass
+        self.quit()
+
+    @classmethod
+    def _ensure_close(cls, self):
+        # needs to be a classmethod so finalize can find the reference
+        logger.info("ensuring close")
+        if (
+            hasattr(self, "service")
+            and hasattr(self.service, "process")
+            and hasattr(self.service.process, "kill")
+        ):
+            self.service.process.kill()
+
+
+def find_chrome_executable():
+    """
+    Finds the chrome, chrome beta, chrome canary, chromium executable
+
+    Returns
+    -------
+    executable_path :  str
+        the full file path to found executable
+
+    """
+    candidates = set()
+    if IS_POSIX:
+        for item in os.environ.get("PATH").split(os.pathsep):
+            for subitem in (
+                "google-chrome",
+                "chromium",
+                "chromium-browser",
+                "chrome",
+                "google-chrome-stable",
+            ):
+                candidates.add(os.sep.join((item, subitem)))
+        if "darwin" in sys.platform:
+            candidates.update(
+                [
+                    "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                    "/Applications/Chromium.app/Contents/MacOS/Chromium",
+                ]
+            )
+    else:
+        for item in map(
+            os.environ.get,
+            ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA", "PROGRAMW6432"),
+        ):
+            if item is not None:
+                for subitem in (
+                    "Google/Chrome/Application",
+                ):
+                    candidates.add(os.sep.join((item, subitem, "chrome.exe")))
+    for candidate in candidates:
+        logger.debug('checking if %s exists and is executable' % candidate)
+        if os.path.exists(candidate) and os.access(candidate, os.X_OK):
+            logger.debug('found! using %s' % candidate)
+            return os.path.normpath(candidate)

src/undetected_chromedriver/cdp.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+import json
+import logging
+
+import requests
+import websockets
+
+
+log = logging.getLogger(__name__)
+
+
+class CDPObject(dict):
+    def __init__(self, *a, **k):
+        super().__init__(*a, **k)
+        self.__dict__ = self
+        for k in self.__dict__:
+            if isinstance(self.__dict__[k], dict):
+                self.__dict__[k] = CDPObject(self.__dict__[k])
+            elif isinstance(self.__dict__[k], list):
+                for i in range(len(self.__dict__[k])):
+                    if isinstance(self.__dict__[k][i], dict):
+                        self.__dict__[k][i] = CDPObject(self)
+
+    def __repr__(self):
+        tpl = f"{self.__class__.__name__}(\n\t{{}}\n\t)"
+        return tpl.format("\n  ".join(f"{k} = {v}" for k, v in self.items()))
+
+
+class PageElement(CDPObject):
+    pass
+
+
+class CDP:
+    log = logging.getLogger("CDP")
+
+    endpoints = CDPObject(
+        {
+            "json": "/json",
+            "protocol": "/json/protocol",
+            "list": "/json/list",
+            "new": "/json/new?{url}",
+            "activate": "/json/activate/{id}",
+            "close": "/json/close/{id}",
+        }
+    )
+
+    def __init__(self, options: "ChromeOptions"):  # noqa
+        self.server_addr = "http://{0}:{1}".format(*options.debugger_address.split(":"))
+
+        self._reqid = 0
+        self._session = requests.Session()
+        self._last_resp = None
+        self._last_json = None
+
+        resp = self.get(self.endpoints.json)  # noqa
+        self.sessionId = resp[0]["id"]
+        self.wsurl = resp[0]["webSocketDebuggerUrl"]
+
+    def tab_activate(self, id=None):
+        if not id:
+            active_tab = self.tab_list()[0]
+            id = active_tab.id  # noqa
+            self.wsurl = active_tab.webSocketDebuggerUrl  # noqa
+        return self.post(self.endpoints["activate"].format(id=id))
+
+    def tab_list(self):
+        retval = self.get(self.endpoints["list"])
+        return [PageElement(o) for o in retval]
+
+    def tab_new(self, url):
+        return self.post(self.endpoints["new"].format(url=url))
+
+    def tab_close_last_opened(self):
+        sessions = self.tab_list()
+        opentabs = [s for s in sessions if s["type"] == "page"]
+        return self.post(self.endpoints["close"].format(id=opentabs[-1]["id"]))
+
+    async def send(self, method: str, params: dict):
+        self._reqid += 1
+        async with websockets.connect(self.wsurl) as ws:
+            await ws.send(
+                json.dumps({"method": method, "params": params, "id": self._reqid})
+            )
+            self._last_resp = await ws.recv()
+            self._last_json = json.loads(self._last_resp)
+            self.log.info(self._last_json)
+
+    def get(self, uri):
+        resp = self._session.get(self.server_addr + uri)
+        try:
+            self._last_resp = resp
+            self._last_json = resp.json()
+        except Exception:
+            return
+        else:
+            return self._last_json
+
+    def post(self, uri, data: dict = None):
+        if not data:
+            data = {}
+        resp = self._session.post(self.server_addr + uri, json=data)
+        try:
+            self._last_resp = resp
+            self._last_json = resp.json()
+        except Exception:
+            return self._last_resp
+
+    @property
+    def last_json(self):
+        return self._last_json

src/undetected_chromedriver/devtool.py

@@ -0,0 +1,193 @@
+import asyncio
+from collections.abc import Mapping
+from collections.abc import Sequence
+from functools import wraps
+import os
+import logging
+import threading
+import time
+import traceback
+from typing import Any
+from typing import Awaitable
+from typing import Callable
+from typing import List
+from typing import Optional
+
+
+class Structure(dict):
+    """
+    This is a dict-like object structure, which you should subclass
+    Only properties defined in the class context are used on initialization.
+
+    See example
+    """
+
+    _store = {}
+
+    def __init__(self, *a, **kw):
+        """
+        Instantiate a new instance.
+
+        :param a:
+        :param kw:
+        """
+
+        super().__init__()
+
+        # auxiliar dict
+        d = dict(*a, **kw)
+        for k, v in d.items():
+            if isinstance(v, Mapping):
+                self[k] = self.__class__(v)
+            elif isinstance(v, Sequence) and not isinstance(v, (str, bytes)):
+                self[k] = [self.__class__(i) for i in v]
+            else:
+                self[k] = v
+        super().__setattr__("__dict__", self)
+
+    def __getattr__(self, item):
+        return getattr(super(), item)
+
+    def __getitem__(self, item):
+        return super().__getitem__(item)
+
+    def __setattr__(self, key, value):
+        self.__setitem__(key, value)
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+
+    def update(self, *a, **kw):
+        super().update(*a, **kw)
+
+    def __eq__(self, other):
+        return frozenset(other.items()) == frozenset(self.items())
+
+    def __hash__(self):
+        return hash(frozenset(self.items()))
+
+    @classmethod
+    def __init_subclass__(cls, **kwargs):
+        cls._store = {}
+
+    def _normalize_strings(self):
+        for k, v in self.copy().items():
+            if isinstance(v, (str)):
+                self[k] = v.strip()
+
+
+def timeout(seconds=3, on_timeout: Optional[Callable[[callable], Any]] = None):
+    def wrapper(func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            def function_reached_timeout():
+                if on_timeout:
+                    on_timeout(func)
+                else:
+                    raise TimeoutError("function call timed out")
+
+            t = threading.Timer(interval=seconds, function=function_reached_timeout)
+            t.start()
+            try:
+                return func(*args, **kwargs)
+            except:
+                t.cancel()
+                raise
+            finally:
+                t.cancel()
+
+        return wrapped
+
+    return wrapper
+
+
+def test():
+    import sys, os
+
+    sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+    import undetected_chromedriver as uc
+    import threading
+
+    def collector(
+        driver: uc.Chrome,
+        stop_event: threading.Event,
+        on_event_coro: Optional[Callable[[List[str]], Awaitable[Any]]] = None,
+        listen_events: Sequence = ("browser", "network", "performance"),
+    ):
+        def threaded(driver, stop_event, on_event_coro):
+            async def _ensure_service_started():
+                while (
+                    getattr(driver, "service", False)
+                    and getattr(driver.service, "process", False)
+                    and driver.service.process.poll()
+                ):
+                    print("waiting for driver service to come back on")
+                    await asyncio.sleep(0.05)
+                    # await asyncio.sleep(driver._delay or .25)
+
+            async def get_log_lines(typ):
+                await _ensure_service_started()
+                return driver.get_log(typ)
+
+            async def looper():
+                while not stop_event.is_set():
+                    log_lines = []
+                    try:
+                        for _ in listen_events:
+                            try:
+                                log_lines += await get_log_lines(_)
+                            except:
+                                if logging.getLogger().getEffectiveLevel() <= 10:
+                                    traceback.print_exc()
+                                continue
+                        if log_lines and on_event_coro:
+                            await on_event_coro(log_lines)
+                    except Exception as e:
+                        if logging.getLogger().getEffectiveLevel() <= 10:
+                            traceback.print_exc()
+
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            loop.run_until_complete(looper())
+
+        t = threading.Thread(target=threaded, args=(driver, stop_event, on_event_coro))
+        t.start()
+
+    async def on_event(data):
+        print("on_event")
+        print("data:", data)
+
+    def func_called(fn):
+        def wrapped(*args, **kwargs):
+            print(
+                "func called! %s  (args: %s, kwargs: %s)" % (fn.__name__, args, kwargs)
+            )
+            while driver.service.process and driver.service.process.poll() is not None:
+                time.sleep(0.1)
+            res = fn(*args, **kwargs)
+            print("func completed! (result: %s)" % res)
+            return res
+
+        return wrapped
+
+    logging.basicConfig(level=10)
+
+    options = uc.ChromeOptions()
+    options.set_capability(
+        "goog:loggingPrefs", {"performance": "ALL", "browser": "ALL", "network": "ALL"}
+    )
+
+    driver = uc.Chrome(version_main=96, options=options)
+
+    # driver.command_executor._request = timeout(seconds=1)(driver.command_executor._request)
+    driver.command_executor._request = func_called(driver.command_executor._request)
+    collector_stop = threading.Event()
+    collector(driver, collector_stop, on_event)
+
+    driver.get("https://nowsecure.nl")
+
+    time.sleep(10)
+
+    if os.name == "nt":
+        driver.close()
+    driver.quit()

src/undetected_chromedriver/dprocess.py

@@ -0,0 +1,77 @@
+import atexit
+import logging
+import multiprocessing
+import os
+import platform
+import signal
+from subprocess import PIPE
+from subprocess import Popen
+import sys
+
+
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+DETACHED_PROCESS = 0x00000008
+
+REGISTERED = []
+
+
+def start_detached(executable, *args):
+    """
+    Starts a fully independent subprocess (with no parent)
+    :param executable: executable
+    :param args: arguments to the executable, eg: ['--param1_key=param1_val', '-vvv' ...]
+    :return: pid of the grandchild process
+    """
+
+    # create pipe
+    reader, writer = multiprocessing.Pipe(False)
+
+    # do not keep reference
+    process = multiprocessing.Process(
+        target=_start_detached,
+        args=(executable, *args),
+        kwargs={"writer": writer},
+        daemon=True,
+    )
+    process.start()
+    process.join()
+    # receive pid from pipe
+    pid = reader.recv()
+    REGISTERED.append(pid)
+    # close pipes
+    writer.close()
+    reader.close()
+    process.close()
+
+    return pid
+
+
+def _start_detached(executable, *args, writer: multiprocessing.Pipe = None):
+    # configure launch
+    kwargs = {}
+    if platform.system() == "Windows":
+        kwargs.update(creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)
+    elif sys.version_info < (3, 2):
+        # assume posix
+        kwargs.update(preexec_fn=os.setsid)
+    else:  # Python 3.2+ and Unix
+        kwargs.update(start_new_session=True)
+
+    # run
+    p = Popen([executable, *args], stdin=PIPE, stdout=PIPE, stderr=PIPE, **kwargs)
+
+    # send pid to pipe
+    writer.send(p.pid)
+    sys.exit()
+
+
+def _cleanup():
+    for pid in REGISTERED:
+        try:
+            logging.getLogger(__name__).debug("cleaning up pid %d " % pid)
+            os.kill(pid, signal.SIGTERM)
+        except:  # noqa
+            pass
+
+
+atexit.register(_cleanup)

src/undetected_chromedriver/options.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+
+import json
+import os
+
+from selenium.webdriver.chromium.options import ChromiumOptions as _ChromiumOptions
+
+
+class ChromeOptions(_ChromiumOptions):
+    _session = None
+    _user_data_dir = None
+
+    @property
+    def user_data_dir(self):
+        return self._user_data_dir
+
+    @user_data_dir.setter
+    def user_data_dir(self, path: str):
+        """
+        Sets the browser profile folder to use, or creates a new profile
+        at given <path>.
+
+        Parameters
+        ----------
+        path: str
+            the path to a chrome profile folder
+            if it does not exist, a new profile will be created at given location
+        """
+        apath = os.path.abspath(path)
+        self._user_data_dir = os.path.normpath(apath)
+
+    @staticmethod
+    def _undot_key(key, value):
+        """turn a (dotted key, value) into a proper nested dict"""
+        if "." in key:
+            key, rest = key.split(".", 1)
+            value = ChromeOptions._undot_key(rest, value)
+        return {key: value}
+
+    @staticmethod
+    def _merge_nested(a, b):
+        """
+        merges b into a
+        leaf values in a are overwritten with values from b
+        """
+        for key in b:
+            if key in a:
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    ChromeOptions._merge_nested(a[key], b[key])
+                    continue
+            a[key] = b[key]
+        return a
+
+    def handle_prefs(self, user_data_dir):
+        prefs = self.experimental_options.get("prefs")
+        if prefs:
+            user_data_dir = user_data_dir or self._user_data_dir
+            default_path = os.path.join(user_data_dir, "Default")
+            os.makedirs(default_path, exist_ok=True)
+
+            # undot prefs dict keys
+            undot_prefs = {}
+            for key, value in prefs.items():
+                undot_prefs = self._merge_nested(
+                    undot_prefs, self._undot_key(key, value)
+                )
+
+            prefs_file = os.path.join(default_path, "Preferences")
+            if os.path.exists(prefs_file):
+                with open(prefs_file, encoding="latin1", mode="r") as f:
+                    undot_prefs = self._merge_nested(json.load(f), undot_prefs)
+
+            with open(prefs_file, encoding="latin1", mode="w") as f:
+                json.dump(undot_prefs, f)
+
+            # remove the experimental_options to avoid an error
+            del self._experimental_options["prefs"]
+
+    @classmethod
+    def from_options(cls, options):
+        o = cls()
+        o.__dict__.update(options.__dict__)
+        return o

src/undetected_chromedriver/patcher.py

@@ -0,0 +1,473 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+from packaging.version import Version as LooseVersion
+import io
+import json
+import logging
+import os
+import pathlib
+import platform
+import random
+import re
+import shutil
+import string
+import subprocess
+import sys
+import time
+from urllib.request import urlopen
+from urllib.request import urlretrieve
+import zipfile
+from multiprocessing import Lock
+
+logger = logging.getLogger(__name__)
+
+IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2", "freebsd"))
+
+
+class Patcher(object):
+    lock = Lock()
+    exe_name = "chromedriver%s"
+
+    platform = sys.platform
+    if platform.endswith("win32"):
+        d = "~/appdata/roaming/undetected_chromedriver"
+    elif "LAMBDA_TASK_ROOT" in os.environ:
+        d = "/tmp/undetected_chromedriver"
+    elif platform.startswith(("linux", "linux2")):
+        d = "~/.local/share/undetected_chromedriver"
+    elif platform.endswith("darwin"):
+        d = "~/Library/Application Support/undetected_chromedriver"
+    else:
+        d = "~/.undetected_chromedriver"
+    data_path = os.path.abspath(os.path.expanduser(d))
+
+    def __init__(
+        self,
+        executable_path=None,
+        force=False,
+        version_main: int = 0,
+        user_multi_procs=False,
+    ):
+        """
+        Args:
+            executable_path: None = automatic
+                             a full file path to the chromedriver executable
+            force: False
+                    terminate processes which are holding lock
+            version_main: 0 = auto
+                specify main chrome version (rounded, ex: 82)
+        """
+        self.force = force
+        self._custom_exe_path = False
+        prefix = "undetected"
+        self.user_multi_procs = user_multi_procs
+
+        try:
+            # Try to convert version_main into an integer
+            version_main_int = int(version_main)
+            # check if version_main_int is less than or equal to e.g 114
+            self.is_old_chromedriver = version_main and version_main_int <= 114
+        except (ValueError,TypeError):
+            # Check not running inside Docker
+            if not os.path.exists("/app/chromedriver"):
+                # If the conversion fails, log an error message
+                logging.info("version_main cannot be converted to an integer")
+            # Set self.is_old_chromedriver to False if the conversion fails
+            self.is_old_chromedriver = False
+
+        # Needs to be called before self.exe_name is accessed
+        self._set_platform_name()
+
+        if not os.path.exists(self.data_path):
+            os.makedirs(self.data_path, exist_ok=True)
+
+        if not executable_path:
+            if sys.platform.startswith("freebsd"):
+                self.executable_path = os.path.join(
+                    self.data_path, self.exe_name
+                )
+            else:
+                self.executable_path = os.path.join(
+                    self.data_path, "_".join([prefix, self.exe_name])
+                )
+
+        if not IS_POSIX:
+            if executable_path:
+                if not executable_path[-4:] == ".exe":
+                    executable_path += ".exe"
+
+        self.zip_path = os.path.join(self.data_path, prefix)
+
+        if not executable_path:
+            if not self.user_multi_procs:
+                self.executable_path = os.path.abspath(
+                    os.path.join(".", self.executable_path)
+                )
+
+        if executable_path:
+            self._custom_exe_path = True
+            self.executable_path = executable_path
+
+        # Set the correct repository to download the Chromedriver from
+        if self.is_old_chromedriver:
+            self.url_repo = "https://chromedriver.storage.googleapis.com"
+        else:
+            self.url_repo = "https://googlechromelabs.github.io/chrome-for-testing"
+
+        self.version_main = version_main
+        self.version_full = None
+
+    def _set_platform_name(self):
+        """
+        Set the platform and exe name based on the platform undetected_chromedriver is running on
+        in order to download the correct chromedriver.
+        """
+        if self.platform.endswith("win32"):
+            self.platform_name = "win32"
+            self.exe_name %= ".exe"
+        if self.platform.endswith(("linux", "linux2")):
+            self.platform_name = "linux64"
+            self.exe_name %= ""
+        if self.platform.endswith("darwin"):
+            if self.is_old_chromedriver:
+                self.platform_name = "mac64"
+            else:
+                self.platform_name = "mac-x64"
+            self.exe_name %= ""
+        if self.platform.startswith("freebsd"):
+            self.platform_name = "freebsd"
+            self.exe_name %= ""
+
+    def auto(self, executable_path=None, force=False, version_main=None, _=None):
+        """
+
+        Args:
+            executable_path:
+            force:
+            version_main:
+
+        Returns:
+
+        """
+        p = pathlib.Path(self.data_path)
+        if self.user_multi_procs:
+            with Lock():
+                files = list(p.rglob("*chromedriver*"))
+                most_recent = max(files, key=lambda f: f.stat().st_mtime)
+                files.remove(most_recent)
+                list(map(lambda f: f.unlink(), files))
+                if self.is_binary_patched(most_recent):
+                    self.executable_path = str(most_recent)
+                    return True
+
+        if executable_path:
+            self.executable_path = executable_path
+            self._custom_exe_path = True
+
+        if self._custom_exe_path:
+            ispatched = self.is_binary_patched(self.executable_path)
+            if not ispatched:
+                return self.patch_exe()
+            else:
+                return
+
+        if version_main:
+            self.version_main = version_main
+        if force is True:
+            self.force = force
+
+
+        if self.platform_name == "freebsd":
+            chromedriver_path = shutil.which("chromedriver")
+
+            if not os.path.isfile(chromedriver_path) or not os.access(chromedriver_path, os.X_OK):
+                logging.error("Chromedriver not installed!")
+                return
+
+            version_path = os.path.join(os.path.dirname(self.executable_path), "version.txt")
+
+            process = os.popen(f'"{chromedriver_path}" --version')
+            chromedriver_version = process.read().split(' ')[1].split(' ')[0]
+            process.close()
+
+            current_version = None
+            if os.path.isfile(version_path) or os.access(version_path, os.X_OK):
+                with open(version_path, 'r') as f:
+                    current_version = f.read()
+
+            if current_version != chromedriver_version:
+                logging.info("Copying chromedriver executable...")
+                shutil.copy(chromedriver_path, self.executable_path)
+                os.chmod(self.executable_path, 0o755)
+
+                with open(version_path, 'w') as f:
+                    f.write(chromedriver_version)
+
+                logging.info("Chromedriver executable copied!")
+        else:
+            try:
+                os.unlink(self.executable_path)
+            except PermissionError:
+                if self.force:
+                    self.force_kill_instances(self.executable_path)
+                    return self.auto(force=not self.force)
+                try:
+                    if self.is_binary_patched():
+                        # assumes already running AND patched
+                        return True
+                except PermissionError:
+                    pass
+                # return False
+            except FileNotFoundError:
+                pass
+
+            release = self.fetch_release_number()
+            self.version_main = release.major
+            self.version_full = release
+            self.unzip_package(self.fetch_package())
+
+        return self.patch()
+
+    def driver_binary_in_use(self, path: str = None) -> bool:
+        """
+        naive test to check if a found chromedriver binary is
+        currently in use
+
+        Args:
+            path: a string or PathLike object to the binary to check.
+                  if not specified, we check use this object's executable_path
+        """
+        if not path:
+            path = self.executable_path
+        p = pathlib.Path(path)
+
+        if not p.exists():
+            raise OSError("file does not exist: %s" % p)
+        try:
+            with open(p, mode="a+b") as fs:
+                exc = []
+                try:
+
+                    fs.seek(0, 0)
+                except PermissionError as e:
+                    exc.append(e)  # since some systems apprently allow seeking
+                    # we conduct another test
+                try:
+                    fs.readline()
+                except PermissionError as e:
+                    exc.append(e)
+
+                if exc:
+
+                    return True
+                return False
+            # ok safe to assume this is in use
+        except Exception as e:
+            # logger.exception("whoops ", e)
+            pass
+
+    def cleanup_unused_files(self):
+        p = pathlib.Path(self.data_path)
+        items = list(p.glob("*undetected*"))
+        for item in items:
+            try:
+                item.unlink()
+            except:
+                pass
+
+    def patch(self):
+        self.patch_exe()
+        return self.is_binary_patched()
+
+    def fetch_release_number(self):
+        """
+        Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
+        :return: version string
+        :rtype: LooseVersion
+        """
+        # Endpoint for old versions of Chromedriver (114 and below)
+        if self.is_old_chromedriver:
+            path = f"/latest_release_{self.version_main}"
+            path = path.upper()
+            logger.debug("getting release number from %s" % path)
+            return LooseVersion(urlopen(self.url_repo + path).read().decode())
+
+        # Endpoint for new versions of Chromedriver (115+)
+        if not self.version_main:
+            # Fetch the latest version
+            path = "/last-known-good-versions-with-downloads.json"
+            logger.debug("getting release number from %s" % path)
+            with urlopen(self.url_repo + path) as conn:
+                response = conn.read().decode()
+
+            last_versions = json.loads(response)
+            return LooseVersion(last_versions["channels"]["Stable"]["version"])
+
+        # Fetch the latest minor version of the major version provided
+        path = "/latest-versions-per-milestone-with-downloads.json"
+        logger.debug("getting release number from %s" % path)
+        with urlopen(self.url_repo + path) as conn:
+            response = conn.read().decode()
+
+        major_versions = json.loads(response)
+        return LooseVersion(major_versions["milestones"][str(self.version_main)]["version"])
+
+    def parse_exe_version(self):
+        with io.open(self.executable_path, "rb") as f:
+            for line in iter(lambda: f.readline(), b""):
+                match = re.search(rb"platform_handle\x00content\x00([0-9.]*)", line)
+                if match:
+                    return LooseVersion(match[1].decode())
+
+    def fetch_package(self):
+        """
+        Downloads ChromeDriver from source
+
+        :return: path to downloaded file
+        """
+        zip_name = f"chromedriver_{self.platform_name}.zip"
+        if self.is_old_chromedriver:
+            download_url = "%s/%s/%s" % (self.url_repo, str(self.version_full), zip_name)
+        else:
+            zip_name = zip_name.replace("_", "-", 1)
+            download_url = "https://storage.googleapis.com/chrome-for-testing-public/%s/%s/%s"
+            download_url %= (str(self.version_full), self.platform_name, zip_name)
+
+        logger.debug("downloading from %s" % download_url)
+        return urlretrieve(download_url)[0]
+
+    def unzip_package(self, fp):
+        """
+        Does what it says
+
+        :return: path to unpacked executable
+        """
+        exe_path = self.exe_name
+        if not self.is_old_chromedriver:
+            # The new chromedriver unzips into its own folder
+            zip_name = f"chromedriver-{self.platform_name}"
+            exe_path = os.path.join(zip_name, self.exe_name)
+
+        logger.debug("unzipping %s" % fp)
+        try:
+            os.unlink(self.zip_path)
+        except (FileNotFoundError, OSError):
+            pass
+
+        os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
+        with zipfile.ZipFile(fp, mode="r") as zf:
+            zf.extractall(self.zip_path)
+        os.rename(os.path.join(self.zip_path, exe_path), self.executable_path)
+        os.remove(fp)
+        shutil.rmtree
+        os.chmod(self.executable_path, 0o755)
+        return self.executable_path
+
+    @staticmethod
+    def force_kill_instances(exe_name):
+        """
+        kills running instances.
+        :param: executable name to kill, may be a path as well
+
+        :return: True on success else False
+        """
+        exe_name = os.path.basename(exe_name)
+        if IS_POSIX:
+            # Using shell=True for pidof, consider a more robust pid finding method if issues arise.
+            # pgrep can be an alternative: ["pgrep", "-f", exe_name]
+            # Or psutil if adding a dependency is acceptable.
+            command = f"pidof {exe_name}"
+            try:
+                result = subprocess.run(command, shell=True, capture_output=True, text=True, check=True)
+                pids = result.stdout.strip().split()
+                if pids:
+                    subprocess.run(["kill", "-9"] + pids, check=False) # Changed from -f -9 to -9 as -f is not standard for kill
+                    return True
+                return False # No PIDs found
+            except subprocess.CalledProcessError: # pidof returns 1 if no process found
+                return False # No process found
+            except Exception as e:
+                logger.debug(f"Error killing process on POSIX: {e}")
+                return False
+        else:
+            try:
+                # TASKKILL /F /IM chromedriver.exe
+                result = subprocess.run(["taskkill", "/f", "/im", exe_name], check=False, capture_output=True)
+                # taskkill returns 0 if process was killed, 128 if not found.
+                return result.returncode == 0
+            except Exception as e:
+                logger.debug(f"Error killing process on Windows: {e}")
+                return False
+
+    @staticmethod
+    def gen_random_cdc():
+        cdc = random.choices(string.ascii_letters, k=27)
+        return "".join(cdc).encode()
+
+    def is_binary_patched(self, executable_path=None):
+        executable_path = executable_path or self.executable_path
+        try:
+            with io.open(executable_path, "rb") as fh:
+                return fh.read().find(b"undetected chromedriver") != -1
+        except FileNotFoundError:
+            return False
+
+    def patch_exe(self):
+        start = time.perf_counter()
+        logger.info("patching driver executable %s" % self.executable_path)
+        with io.open(self.executable_path, "r+b") as fh:
+            content = fh.read()
+            # match_injected_codeblock = re.search(rb"{window.*;}", content)
+            match_injected_codeblock = re.search(rb"\{window\.cdc.*?;\}", content)
+            if match_injected_codeblock:
+                target_bytes = match_injected_codeblock[0]
+                new_target_bytes = (
+                    b'{console.log("undetected chromedriver 1337!")}'.ljust(
+                        len(target_bytes), b" "
+                    )
+                )
+                new_content = content.replace(target_bytes, new_target_bytes)
+                if new_content == content:
+                    logger.warning(
+                        "something went wrong patching the driver binary. could not find injection code block"
+                    )
+                else:
+                    logger.debug(
+                        "found block:\n%s\nreplacing with:\n%s"
+                        % (target_bytes, new_target_bytes)
+                    )
+                fh.seek(0)
+                fh.write(new_content)
+        logger.debug(
+            "patching took us {:.2f} seconds".format(time.perf_counter() - start)
+        )
+
+    def __repr__(self):
+        return "{0:s}({1:s})".format(
+            self.__class__.__name__,
+            self.executable_path,
+        )
+
+    def __del__(self):
+        if self._custom_exe_path:
+            # if the driver binary is specified by user
+            # we assume it is important enough to not delete it
+            return
+        else:
+            timeout = 3  # stop trying after this many seconds
+            t = time.monotonic()
+            now = lambda: time.monotonic()
+            while now() - t > timeout:
+                # we don't want to wait until the end of time
+                try:
+                    if self.user_multi_procs:
+                        break
+                    os.unlink(self.executable_path)
+                    logger.debug("successfully unlinked %s" % self.executable_path)
+                    break
+                except (OSError, RuntimeError, PermissionError):
+                    time.sleep(0.01)
+                    continue
+                except FileNotFoundError:
+                    break

src/undetected_chromedriver/reactor.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+import asyncio
+import json
+import logging
+import threading
+
+
+logger = logging.getLogger(__name__)
+
+
+class Reactor(threading.Thread):
+    def __init__(self, driver: "Chrome"):
+        super().__init__()
+
+        self.driver = driver
+        self.loop = asyncio.new_event_loop()
+
+        self.lock = threading.Lock()
+        self.event = threading.Event()
+        self.daemon = True
+        self.handlers = {}
+
+    def add_event_handler(self, method_name, callback: callable):
+        """
+
+        Parameters
+        ----------
+        event_name: str
+            example "Network.responseReceived"
+
+        callback: callable
+            callable which accepts 1 parameter: the message object dictionary
+
+        Returns
+        -------
+
+        """
+        with self.lock:
+            self.handlers[method_name.lower()] = callback
+
+    @property
+    def running(self):
+        return not self.event.is_set()
+
+    def run(self):
+        try:
+            asyncio.set_event_loop(self.loop)
+            self.loop.run_until_complete(self.listen())
+        except Exception as e:
+            logger.warning("Reactor.run() => %s", e)
+
+    async def _wait_service_started(self):
+        while True:
+            with self.lock:
+                if (
+                    getattr(self.driver, "service", None)
+                    and getattr(self.driver.service, "process", None)
+                    and self.driver.service.process.poll()
+                ):
+                    await asyncio.sleep(self.driver._delay or 0.25)
+                else:
+                    break
+
+    async def listen(self):
+        while self.running:
+            await self._wait_service_started()
+            await asyncio.sleep(1)
+
+            try:
+                with self.lock:
+                    log_entries = self.driver.get_log("performance")
+
+                for entry in log_entries:
+                    try:
+                        obj_serialized: str = entry.get("message")
+                        obj = json.loads(obj_serialized)
+                        message = obj.get("message")
+                        method = message.get("method")
+
+                        if "*" in self.handlers:
+                            await self.loop.run_in_executor(
+                                None, self.handlers["*"], message
+                            )
+                        elif method.lower() in self.handlers:
+                            await self.loop.run_in_executor(
+                                None, self.handlers[method.lower()], message
+                            )
+
+                        # print(type(message), message)
+                    except Exception as e:
+                        raise e from None
+
+            except Exception as e:
+                if "invalid session id" in str(e):
+                    pass
+                else:
+                    logging.debug("exception ignored :", e)

src/undetected_chromedriver/webelement.py

@@ -0,0 +1,86 @@
+from typing import List
+
+from selenium.webdriver.common.by import By
+import selenium.webdriver.remote.webelement
+
+
+class WebElement(selenium.webdriver.remote.webelement.WebElement):
+    def click_safe(self):
+        super().click()
+        self._parent.reconnect(0.1)
+
+    def children(
+        self, tag=None, recursive=False
+    ) -> List[selenium.webdriver.remote.webelement.WebElement]:
+        """
+        returns direct child elements of current element
+        :param tag: str,  if supplied, returns <tag> nodes only
+        """
+        script = "return [... arguments[0].children]"
+        if tag:
+            script += ".filter( node => node.tagName === '%s')" % tag.upper()
+        if recursive:
+            return list(_recursive_children(self, tag))
+        return list(self._parent.execute_script(script, self))
+
+
+class UCWebElement(WebElement):
+    """
+    Custom WebElement class which makes it easier to view elements when
+    working in an interactive environment.
+
+    standard webelement repr:
+    <selenium.webdriver.remote.webelement.WebElement (session="85ff0f671512fa535630e71ee951b1f2", element="6357cb55-92c3-4c0f-9416-b174f9c1b8c4")>
+
+    using this WebElement class:
+    <WebElement(<a class="mobile-show-inline-block mc-update-infos init-ok" href="#" id="main-cat-switcher-mobile">)>
+
+    """
+
+    def __init__(self, parent, id_):
+        super().__init__(parent, id_)
+        self._attrs = None
+
+    @property
+    def attrs(self):
+        if not self._attrs:
+            self._attrs = self._parent.execute_script(
+                """
+                var items = {}; 
+                for (index = 0; index < arguments[0].attributes.length; ++index) 
+                {
+                 items[arguments[0].attributes[index].name] = arguments[0].attributes[index].value 
+                }; 
+                return items;
+                """,
+                self,
+            )
+        return self._attrs
+
+    def __repr__(self):
+        strattrs = " ".join([f'{k}="{v}"' for k, v in self.attrs.items()])
+        if strattrs:
+            strattrs = " " + strattrs
+        return f"{self.__class__.__name__} <{self.tag_name}{strattrs}>"
+
+
+def _recursive_children(element, tag: str = None, _results=None):
+    """
+    returns all children of <element> recursively
+
+    :param element: `WebElement` object.
+            find children below this <element>
+
+    :param tag: str = None.
+            if provided, return only <tag> elements. example: 'a', or 'img'
+    :param _results: do not use!
+    """
+    results = _results or set()
+    for element in element.children():
+        if tag:
+            if element.tag_name == tag:
+                results.add(element)
+        else:
+            results.add(element)
+        results |= _recursive_children(element, tag, results)
+    return results

src/utils.py

@@ -0,0 +1,347 @@
+import json
+import logging
+import os
+import platform
+import re
+import shutil
+import sys
+import tempfile
+import urllib.parse
+
+from selenium.webdriver.chrome.webdriver import WebDriver
+import undetected_chromedriver as uc
+
+FLARESOLVERR_VERSION = None
+PLATFORM_VERSION = None
+CHROME_EXE_PATH = None
+CHROME_MAJOR_VERSION = None
+USER_AGENT = None
+XVFB_DISPLAY = None
+PATCHED_DRIVER_PATH = None
+
+
+def get_config_log_html() -> bool:
+    return os.environ.get('LOG_HTML', 'false').lower() == 'true'
+
+
+def get_config_headless() -> bool:
+    return os.environ.get('HEADLESS', 'true').lower() == 'true'
+
+
+def get_config_disable_media() -> bool:
+    return os.environ.get('DISABLE_MEDIA', 'false').lower() == 'true'
+
+
+def get_flaresolverr_version() -> str:
+    global FLARESOLVERR_VERSION
+    if FLARESOLVERR_VERSION is not None:
+        return FLARESOLVERR_VERSION
+
+    package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json')
+    if not os.path.isfile(package_path):
+        package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json')
+    with open(package_path) as f:
+        FLARESOLVERR_VERSION = json.loads(f.read())['version']
+        return FLARESOLVERR_VERSION
+
+def get_current_platform() -> str:
+    global PLATFORM_VERSION
+    if PLATFORM_VERSION is not None:
+        return PLATFORM_VERSION
+    PLATFORM_VERSION = os.name
+    return PLATFORM_VERSION
+
+
+def create_proxy_extension(proxy: dict) -> str:
+    parsed_url = urllib.parse.urlparse(proxy['url'])
+    scheme = parsed_url.scheme
+    host = parsed_url.hostname
+    port = parsed_url.port
+    username = proxy['username']
+    password = proxy['password']
+    manifest_json = """
+    {
+        "version": "1.0.0",
+        "manifest_version": 3,
+        "name": "Chrome Proxy",
+        "permissions": [
+            "proxy",
+            "tabs",
+            "storage",
+            "webRequest",
+            "webRequestAuthProvider"
+        ],
+        "host_permissions": [
+          "<all_urls>"
+        ],
+        "background": {
+          "service_worker": "background.js"
+        },
+        "minimum_chrome_version": "76.0.0"
+    }
+    """
+
+    background_js = """
+    var config = {
+        mode: "fixed_servers",
+        rules: {
+            singleProxy: {
+                scheme: "%s",
+                host: "%s",
+                port: %d
+            },
+            bypassList: ["localhost"]
+        }
+    };
+
+    chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
+
+    function callbackFn(details) {
+        return {
+            authCredentials: {
+                username: "%s",
+                password: "%s"
+            }
+        };
+    }
+
+    chrome.webRequest.onAuthRequired.addListener(
+        callbackFn,
+        { urls: ["<all_urls>"] },
+        ['blocking']
+    );
+    """ % (
+        scheme,
+        host,
+        port,
+        username,
+        password
+    )
+
+    proxy_extension_dir = tempfile.mkdtemp()
+
+    with open(os.path.join(proxy_extension_dir, "manifest.json"), "w") as f:
+        f.write(manifest_json)
+
+    with open(os.path.join(proxy_extension_dir, "background.js"), "w") as f:
+        f.write(background_js)
+
+    return proxy_extension_dir
+
+
+def get_webdriver(proxy: dict = None) -> WebDriver:
+    global PATCHED_DRIVER_PATH, USER_AGENT
+    logging.debug('Launching web browser...')
+
+    # undetected_chromedriver
+    options = uc.ChromeOptions()
+    options.add_argument('--no-sandbox')
+    options.add_argument('--window-size=1920,1080')
+    options.add_argument('--disable-search-engine-choice-screen')
+    # todo: this param shows a warning in chrome head-full
+    options.add_argument('--disable-setuid-sandbox')
+    options.add_argument('--disable-dev-shm-usage')
+    # this option removes the zygote sandbox (it seems that the resolution is a bit faster)
+    options.add_argument('--no-zygote')
+    # attempt to fix Docker ARM32 build
+    IS_ARMARCH = platform.machine().startswith(('arm', 'aarch'))
+    if IS_ARMARCH:
+        options.add_argument('--disable-gpu-sandbox')
+    options.add_argument('--ignore-certificate-errors')
+    options.add_argument('--ignore-ssl-errors')
+
+    language = os.environ.get('LANG', None)
+    if language is not None:
+        options.add_argument('--accept-lang=%s' % language)
+
+    # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
+    if USER_AGENT is not None:
+        options.add_argument('--user-agent=%s' % USER_AGENT)
+
+    proxy_extension_dir = None
+    if proxy and all(key in proxy for key in ['url', 'username', 'password']):
+        proxy_extension_dir = create_proxy_extension(proxy)
+        options.add_argument("--disable-features=DisableLoadExtensionCommandLineSwitch")
+        options.add_argument("--load-extension=%s" % os.path.abspath(proxy_extension_dir))
+    elif proxy and 'url' in proxy:
+        proxy_url = proxy['url']
+        logging.debug("Using webdriver proxy: %s", proxy_url)
+        options.add_argument('--proxy-server=%s' % proxy_url)
+
+    # note: headless mode is detected (headless = True)
+    # we launch the browser in head-full mode with the window hidden
+    windows_headless = False
+    if get_config_headless():
+        if os.name == 'nt':
+            windows_headless = True
+        else:
+            start_xvfb_display()
+    # For normal headless mode:
+    # options.add_argument('--headless')
+
+    # if we are inside the Docker container, we avoid downloading the driver
+    driver_exe_path = None
+    version_main = None
+    if os.path.exists("/app/chromedriver"):
+        # running inside Docker
+        driver_exe_path = "/app/chromedriver"
+    else:
+        version_main = get_chrome_major_version()
+        if PATCHED_DRIVER_PATH is not None:
+            driver_exe_path = PATCHED_DRIVER_PATH
+
+    # detect chrome path
+    browser_executable_path = get_chrome_exe_path()
+
+    # downloads and patches the chromedriver
+    # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
+    try:
+        driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path,
+                           driver_executable_path=driver_exe_path, version_main=version_main,
+                           windows_headless=windows_headless, headless=get_config_headless())
+    except Exception as e:
+        logging.error("Error starting Chrome: %s" % e)
+        # No point in continuing if we cannot retrieve the driver
+        raise e
+
+    # save the patched driver to avoid re-downloads
+    if driver_exe_path is None:
+        PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
+        if PATCHED_DRIVER_PATH != driver.patcher.executable_path:
+            shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
+
+    # clean up proxy extension directory
+    if proxy_extension_dir is not None:
+        shutil.rmtree(proxy_extension_dir)
+
+    # selenium vanilla
+    # options = webdriver.ChromeOptions()
+    # options.add_argument('--no-sandbox')
+    # options.add_argument('--window-size=1920,1080')
+    # options.add_argument('--disable-setuid-sandbox')
+    # options.add_argument('--disable-dev-shm-usage')
+    # driver = webdriver.Chrome(options=options)
+
+    return driver
+
+
+def get_chrome_exe_path() -> str:
+    global CHROME_EXE_PATH
+    if CHROME_EXE_PATH is not None:
+        return CHROME_EXE_PATH
+    # linux pyinstaller bundle
+    chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
+    if os.path.exists(chrome_path):
+        if not os.access(chrome_path, os.X_OK):
+            raise Exception(f'Chrome binary "{chrome_path}" is not executable. '
+                            f'Please, extract the archive with "tar xzf <file.tar.gz>".')
+        CHROME_EXE_PATH = chrome_path
+        return CHROME_EXE_PATH
+    # windows pyinstaller bundle
+    chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
+    if os.path.exists(chrome_path):
+        CHROME_EXE_PATH = chrome_path
+        return CHROME_EXE_PATH
+    # system
+    CHROME_EXE_PATH = uc.find_chrome_executable()
+    return CHROME_EXE_PATH
+
+
+def get_chrome_major_version() -> str:
+    global CHROME_MAJOR_VERSION
+    if CHROME_MAJOR_VERSION is not None:
+        return CHROME_MAJOR_VERSION
+
+    if os.name == 'nt':
+        # Example: '104.0.5112.79'
+        try:
+            complete_version = extract_version_nt_executable(get_chrome_exe_path())
+        except Exception:
+            try:
+                complete_version = extract_version_nt_registry()
+            except Exception:
+                # Example: '104.0.5112.79'
+                complete_version = extract_version_nt_folder()
+    else:
+        chrome_path = get_chrome_exe_path()
+        process = os.popen(f'"{chrome_path}" --version')
+        # Example 1: 'Chromium 104.0.5112.79 Arch Linux\n'
+        # Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n'
+        complete_version = process.read()
+        process.close()
+
+    CHROME_MAJOR_VERSION = complete_version.split('.')[0].split(' ')[-1]
+    return CHROME_MAJOR_VERSION
+
+
+def extract_version_nt_executable(exe_path: str) -> str:
+    import pefile
+    pe = pefile.PE(exe_path, fast_load=True)
+    pe.parse_data_directories(
+        directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]
+    )
+    return pe.FileInfo[0][0].StringTable[0].entries[b"FileVersion"].decode('utf-8')
+
+
+def extract_version_nt_registry() -> str:
+    stream = os.popen(
+        'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
+    output = stream.read()
+    google_version = ''
+    for letter in output[output.rindex('DisplayVersion    REG_SZ') + 24:]:
+        if letter != '\n':
+            google_version += letter
+        else:
+            break
+    return google_version.strip()
+
+
+def extract_version_nt_folder() -> str:
+    # Check if the Chrome folder exists in the x32 or x64 Program Files folders.
+    for i in range(2):
+        path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'
+        if os.path.isdir(path):
+            paths = [f.path for f in os.scandir(path) if f.is_dir()]
+            for path in paths:
+                filename = os.path.basename(path)
+                pattern = r'\d+\.\d+\.\d+\.\d+'
+                match = re.search(pattern, filename)
+                if match and match.group():
+                    # Found a Chrome version.
+                    return match.group(0)
+    return ''
+
+
+def get_user_agent(driver=None) -> str:
+    global USER_AGENT
+    if USER_AGENT is not None:
+        return USER_AGENT
+
+    try:
+        if driver is None:
+            driver = get_webdriver()
+        USER_AGENT = driver.execute_script("return navigator.userAgent")
+        # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
+        USER_AGENT = re.sub('HEADLESS', '', USER_AGENT, flags=re.IGNORECASE)
+        return USER_AGENT
+    except Exception as e:
+        raise Exception("Error getting browser User-Agent. " + str(e))
+    finally:
+        if driver is not None:
+            if PLATFORM_VERSION == "nt":
+                driver.close()
+            driver.quit()
+
+
+def start_xvfb_display():
+    global XVFB_DISPLAY
+    if XVFB_DISPLAY is None:
+        from xvfbwrapper import Xvfb
+        XVFB_DISPLAY = Xvfb()
+        XVFB_DISPLAY.start()
+
+
+def object_to_dict(_object):
+    json_dict = json.loads(json.dumps(_object, default=lambda o: o.__dict__))
+    # remove hidden fields
+    return {k: v for k, v in json_dict.items() if not k.startswith('__')}