Spaces:
Paused
Paused
Update uc_server.py
Browse files- uc_server.py +21 -41
uc_server.py
CHANGED
|
@@ -1,17 +1,9 @@
|
|
| 1 |
-
from flask import Flask
|
| 2 |
-
from flask import request
|
| 3 |
import undetected_chromedriver as uc
|
| 4 |
import re
|
| 5 |
import os
|
| 6 |
from threading import Timer
|
| 7 |
-
|
| 8 |
-
# /start_browser
|
| 9 |
-
# /stop_browser
|
| 10 |
-
# /text?url=https://api.investing.com/api/financialdata/8849/historical/chart/?interval=PT1M&pointscount=60
|
| 11 |
-
# encoded: https%3A%2F%2Fapi.investing.com%2Fapi%2Ffinancialdata%2F8849%2Fhistorical%2Fchart%2F%3Finterval%3DPT1M%26pointscount%3D60
|
| 12 |
-
|
| 13 |
-
# /fetch?url=https://api.investing.com/api/financialdata/historical/1?start-date=2024-02-15&end-date=2029-02-15&time-frame=Daily&add-missing-rows=false
|
| 14 |
-
# encoded: https%3A%2F%2Fapi.investing.com%2Fapi%2Ffinancialdata%2Fhistorical%2F1%3Fstart-date%3D2024-02-15%26end-date%3D2029-02-15%26time-frame%3DDaily%26add-missing-rows%3Dfalse
|
| 15 |
|
| 16 |
driver = None
|
| 17 |
XVFB_DISPLAY = None
|
|
@@ -32,12 +24,10 @@ def get_chrome_exe_path() -> str:
|
|
| 32 |
global CHROME_EXE_PATH
|
| 33 |
if CHROME_EXE_PATH is not None:
|
| 34 |
return CHROME_EXE_PATH
|
| 35 |
-
# linux pyinstaller bundle
|
| 36 |
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
|
| 37 |
if os.path.exists(chrome_path):
|
| 38 |
CHROME_EXE_PATH = chrome_path
|
| 39 |
return CHROME_EXE_PATH
|
| 40 |
-
# system
|
| 41 |
CHROME_EXE_PATH = uc.find_chrome_executable()
|
| 42 |
return CHROME_EXE_PATH
|
| 43 |
|
|
@@ -50,34 +40,23 @@ def _start_browser():
|
|
| 50 |
|
| 51 |
_start_xvfb_display()
|
| 52 |
|
| 53 |
-
# https://github.com/FlareSolverr/FlareSolverr/blob/043f18b231b4f409080b2b5c4421ce0f4cac7dec/src/utils.py
|
| 54 |
options = uc.ChromeOptions()
|
| 55 |
options.add_argument('--no-sandbox')
|
| 56 |
options.add_argument('--window-size=1920,1080')
|
| 57 |
-
# todo: this param shows a warning in chrome head-full
|
| 58 |
options.add_argument('--disable-setuid-sandbox')
|
| 59 |
options.add_argument('--disable-dev-shm-usage')
|
| 60 |
-
# this option removes the zygote sandbox (it seems that the resolution is a bit faster)
|
| 61 |
options.add_argument('--no-zygote')
|
| 62 |
-
# attempt to fix Docker ARM32 build
|
| 63 |
options.add_argument('--disable-gpu-sandbox')
|
| 64 |
options.add_argument('--disable-software-rasterizer')
|
| 65 |
options.add_argument('--ignore-certificate-errors')
|
| 66 |
options.add_argument('--ignore-ssl-errors')
|
| 67 |
-
# fix GL errors in ASUSTOR NAS
|
| 68 |
-
# https://github.com/FlareSolverr/FlareSolverr/issues/782
|
| 69 |
-
# https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
|
| 70 |
-
# https://peter.sh/experiments/chromium-command-line-switches/#use-gl
|
| 71 |
options.add_argument('--use-gl=swiftshader')
|
| 72 |
-
#options.add_argument("--headless=new")
|
| 73 |
if USER_AGENT is not None:
|
| 74 |
options.add_argument('--user-agent=%s' % USER_AGENT)
|
| 75 |
language = os.environ.get('LANG', None)
|
| 76 |
if language is not None:
|
| 77 |
options.add_argument('--lang=%s' % language)
|
| 78 |
-
|
| 79 |
-
# added by me
|
| 80 |
-
options.add_argument(' --disable-web-security') # allow cross origin
|
| 81 |
|
| 82 |
driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
|
| 83 |
|
|
@@ -87,33 +66,23 @@ def _start_browser():
|
|
| 87 |
app.logger.info(USER_AGENT)
|
| 88 |
|
| 89 |
driver.quit()
|
| 90 |
-
#restart with user agent
|
| 91 |
options = uc.ChromeOptions()
|
| 92 |
options.add_argument('--no-sandbox')
|
| 93 |
options.add_argument('--window-size=1920,1080')
|
| 94 |
-
# todo: this param shows a warning in chrome head-full
|
| 95 |
options.add_argument('--disable-setuid-sandbox')
|
| 96 |
options.add_argument('--disable-dev-shm-usage')
|
| 97 |
-
# this option removes the zygote sandbox (it seems that the resolution is a bit faster)
|
| 98 |
options.add_argument('--no-zygote')
|
| 99 |
-
# attempt to fix Docker ARM32 build
|
| 100 |
options.add_argument('--disable-gpu-sandbox')
|
| 101 |
options.add_argument('--disable-software-rasterizer')
|
| 102 |
options.add_argument('--ignore-certificate-errors')
|
| 103 |
options.add_argument('--ignore-ssl-errors')
|
| 104 |
-
# fix GL errors in ASUSTOR NAS
|
| 105 |
-
# https://github.com/FlareSolverr/FlareSolverr/issues/782
|
| 106 |
-
# https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
|
| 107 |
-
# https://peter.sh/experiments/chromium-command-line-switches/#use-gl
|
| 108 |
options.add_argument('--use-gl=swiftshader')
|
| 109 |
if USER_AGENT is not None:
|
| 110 |
options.add_argument('--user-agent=%s' % USER_AGENT)
|
| 111 |
language = os.environ.get('LANG', None)
|
| 112 |
if language is not None:
|
| 113 |
options.add_argument('--lang=%s' % language)
|
| 114 |
-
|
| 115 |
-
# added by me
|
| 116 |
-
options.add_argument(' --disable-web-security') # allow cross origin
|
| 117 |
|
| 118 |
driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
|
| 119 |
|
|
@@ -121,7 +90,6 @@ def _start_browser():
|
|
| 121 |
|
| 122 |
app.logger.info("browser started")
|
| 123 |
|
| 124 |
-
|
| 125 |
def _stop_browser():
|
| 126 |
global driver
|
| 127 |
driver.quit()
|
|
@@ -132,7 +100,7 @@ def _reset_stop_timer():
|
|
| 132 |
global timer
|
| 133 |
if timer is not None:
|
| 134 |
timer.cancel()
|
| 135 |
-
timer = Timer(5*60, _stop_browser)
|
| 136 |
timer.start()
|
| 137 |
|
| 138 |
@app.route("/")
|
|
@@ -160,12 +128,25 @@ def text():
|
|
| 160 |
url = request.args.get('url', '')
|
| 161 |
driver.get(url)
|
| 162 |
text = driver.page_source
|
| 163 |
-
#driver.close()
|
| 164 |
return text
|
| 165 |
|
| 166 |
@app.route("/screenshot")
|
| 167 |
def screenshot():
|
| 168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
|
| 170 |
@app.route("/evaluate")
|
| 171 |
def evaluate():
|
|
@@ -181,11 +162,10 @@ def fetch():
|
|
| 181 |
|
| 182 |
url = request.args.get('url', '')
|
| 183 |
|
| 184 |
-
#driver.get('https://example.com')
|
| 185 |
driver.get('https://i-invdn-com.investing.com/redesign/images/seo/investing_300X300.png')
|
| 186 |
|
| 187 |
script = """
|
| 188 |
-
var callback = arguments[arguments.length - 1];
|
| 189 |
(async function(){
|
| 190 |
try {
|
| 191 |
let res = await fetch('%s', {headers:{'domain-id':'www'}});
|
|
|
|
| 1 |
+
from flask import Flask, request, send_file
|
|
|
|
| 2 |
import undetected_chromedriver as uc
|
| 3 |
import re
|
| 4 |
import os
|
| 5 |
from threading import Timer
|
| 6 |
+
import time
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
driver = None
|
| 9 |
XVFB_DISPLAY = None
|
|
|
|
| 24 |
global CHROME_EXE_PATH
|
| 25 |
if CHROME_EXE_PATH is not None:
|
| 26 |
return CHROME_EXE_PATH
|
|
|
|
| 27 |
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
|
| 28 |
if os.path.exists(chrome_path):
|
| 29 |
CHROME_EXE_PATH = chrome_path
|
| 30 |
return CHROME_EXE_PATH
|
|
|
|
| 31 |
CHROME_EXE_PATH = uc.find_chrome_executable()
|
| 32 |
return CHROME_EXE_PATH
|
| 33 |
|
|
|
|
| 40 |
|
| 41 |
_start_xvfb_display()
|
| 42 |
|
|
|
|
| 43 |
options = uc.ChromeOptions()
|
| 44 |
options.add_argument('--no-sandbox')
|
| 45 |
options.add_argument('--window-size=1920,1080')
|
|
|
|
| 46 |
options.add_argument('--disable-setuid-sandbox')
|
| 47 |
options.add_argument('--disable-dev-shm-usage')
|
|
|
|
| 48 |
options.add_argument('--no-zygote')
|
|
|
|
| 49 |
options.add_argument('--disable-gpu-sandbox')
|
| 50 |
options.add_argument('--disable-software-rasterizer')
|
| 51 |
options.add_argument('--ignore-certificate-errors')
|
| 52 |
options.add_argument('--ignore-ssl-errors')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
options.add_argument('--use-gl=swiftshader')
|
|
|
|
| 54 |
if USER_AGENT is not None:
|
| 55 |
options.add_argument('--user-agent=%s' % USER_AGENT)
|
| 56 |
language = os.environ.get('LANG', None)
|
| 57 |
if language is not None:
|
| 58 |
options.add_argument('--lang=%s' % language)
|
| 59 |
+
options.add_argument(' --disable-web-security')
|
|
|
|
|
|
|
| 60 |
|
| 61 |
driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
|
| 62 |
|
|
|
|
| 66 |
app.logger.info(USER_AGENT)
|
| 67 |
|
| 68 |
driver.quit()
|
|
|
|
| 69 |
options = uc.ChromeOptions()
|
| 70 |
options.add_argument('--no-sandbox')
|
| 71 |
options.add_argument('--window-size=1920,1080')
|
|
|
|
| 72 |
options.add_argument('--disable-setuid-sandbox')
|
| 73 |
options.add_argument('--disable-dev-shm-usage')
|
|
|
|
| 74 |
options.add_argument('--no-zygote')
|
|
|
|
| 75 |
options.add_argument('--disable-gpu-sandbox')
|
| 76 |
options.add_argument('--disable-software-rasterizer')
|
| 77 |
options.add_argument('--ignore-certificate-errors')
|
| 78 |
options.add_argument('--ignore-ssl-errors')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
options.add_argument('--use-gl=swiftshader')
|
| 80 |
if USER_AGENT is not None:
|
| 81 |
options.add_argument('--user-agent=%s' % USER_AGENT)
|
| 82 |
language = os.environ.get('LANG', None)
|
| 83 |
if language is not None:
|
| 84 |
options.add_argument('--lang=%s' % language)
|
| 85 |
+
options.add_argument(' --disable-web-security')
|
|
|
|
|
|
|
| 86 |
|
| 87 |
driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
|
| 88 |
|
|
|
|
| 90 |
|
| 91 |
app.logger.info("browser started")
|
| 92 |
|
|
|
|
| 93 |
def _stop_browser():
|
| 94 |
global driver
|
| 95 |
driver.quit()
|
|
|
|
| 100 |
global timer
|
| 101 |
if timer is not None:
|
| 102 |
timer.cancel()
|
| 103 |
+
timer = Timer(5*60, _stop_browser)
|
| 104 |
timer.start()
|
| 105 |
|
| 106 |
@app.route("/")
|
|
|
|
| 128 |
url = request.args.get('url', '')
|
| 129 |
driver.get(url)
|
| 130 |
text = driver.page_source
|
|
|
|
| 131 |
return text
|
| 132 |
|
| 133 |
@app.route("/screenshot")
|
| 134 |
def screenshot():
|
| 135 |
+
global driver
|
| 136 |
+
if driver is None:
|
| 137 |
+
_start_browser()
|
| 138 |
+
else:
|
| 139 |
+
_reset_stop_timer()
|
| 140 |
+
|
| 141 |
+
url = request.args.get('url', 'https://www.youtube.com')
|
| 142 |
+
driver.get(url)
|
| 143 |
+
|
| 144 |
+
time.sleep(5) # Adjust the sleep time as needed
|
| 145 |
+
|
| 146 |
+
screenshot_path = "/tmp/screenshot.png"
|
| 147 |
+
driver.save_screenshot(screenshot_path)
|
| 148 |
+
|
| 149 |
+
return send_file(screenshot_path, mimetype='image/png')
|
| 150 |
|
| 151 |
@app.route("/evaluate")
|
| 152 |
def evaluate():
|
|
|
|
| 162 |
|
| 163 |
url = request.args.get('url', '')
|
| 164 |
|
|
|
|
| 165 |
driver.get('https://i-invdn-com.investing.com/redesign/images/seo/investing_300X300.png')
|
| 166 |
|
| 167 |
script = """
|
| 168 |
+
var callback = arguments[arguments.length - 1];
|
| 169 |
(async function(){
|
| 170 |
try {
|
| 171 |
let res = await fetch('%s', {headers:{'domain-id':'www'}});
|