AZLABS commited on
Commit
20fe9b2
·
verified ·
1 Parent(s): 9bf15a6

Update uc_server.py

Browse files
Files changed (1) hide show
  1. uc_server.py +21 -41
uc_server.py CHANGED
@@ -1,17 +1,9 @@
1
- from flask import Flask
2
- from flask import request
3
  import undetected_chromedriver as uc
4
  import re
5
  import os
6
  from threading import Timer
7
-
8
- # /start_browser
9
- # /stop_browser
10
- # /text?url=https://api.investing.com/api/financialdata/8849/historical/chart/?interval=PT1M&pointscount=60
11
- # encoded: https%3A%2F%2Fapi.investing.com%2Fapi%2Ffinancialdata%2F8849%2Fhistorical%2Fchart%2F%3Finterval%3DPT1M%26pointscount%3D60
12
-
13
- # /fetch?url=https://api.investing.com/api/financialdata/historical/1?start-date=2024-02-15&end-date=2029-02-15&time-frame=Daily&add-missing-rows=false
14
- # encoded: https%3A%2F%2Fapi.investing.com%2Fapi%2Ffinancialdata%2Fhistorical%2F1%3Fstart-date%3D2024-02-15%26end-date%3D2029-02-15%26time-frame%3DDaily%26add-missing-rows%3Dfalse
15
 
16
  driver = None
17
  XVFB_DISPLAY = None
@@ -32,12 +24,10 @@ def get_chrome_exe_path() -> str:
32
  global CHROME_EXE_PATH
33
  if CHROME_EXE_PATH is not None:
34
  return CHROME_EXE_PATH
35
- # linux pyinstaller bundle
36
  chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
37
  if os.path.exists(chrome_path):
38
  CHROME_EXE_PATH = chrome_path
39
  return CHROME_EXE_PATH
40
- # system
41
  CHROME_EXE_PATH = uc.find_chrome_executable()
42
  return CHROME_EXE_PATH
43
 
@@ -50,34 +40,23 @@ def _start_browser():
50
 
51
  _start_xvfb_display()
52
 
53
- # https://github.com/FlareSolverr/FlareSolverr/blob/043f18b231b4f409080b2b5c4421ce0f4cac7dec/src/utils.py
54
  options = uc.ChromeOptions()
55
  options.add_argument('--no-sandbox')
56
  options.add_argument('--window-size=1920,1080')
57
- # todo: this param shows a warning in chrome head-full
58
  options.add_argument('--disable-setuid-sandbox')
59
  options.add_argument('--disable-dev-shm-usage')
60
- # this option removes the zygote sandbox (it seems that the resolution is a bit faster)
61
  options.add_argument('--no-zygote')
62
- # attempt to fix Docker ARM32 build
63
  options.add_argument('--disable-gpu-sandbox')
64
  options.add_argument('--disable-software-rasterizer')
65
  options.add_argument('--ignore-certificate-errors')
66
  options.add_argument('--ignore-ssl-errors')
67
- # fix GL errors in ASUSTOR NAS
68
- # https://github.com/FlareSolverr/FlareSolverr/issues/782
69
- # https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
70
- # https://peter.sh/experiments/chromium-command-line-switches/#use-gl
71
  options.add_argument('--use-gl=swiftshader')
72
- #options.add_argument("--headless=new")
73
  if USER_AGENT is not None:
74
  options.add_argument('--user-agent=%s' % USER_AGENT)
75
  language = os.environ.get('LANG', None)
76
  if language is not None:
77
  options.add_argument('--lang=%s' % language)
78
-
79
- # added by me
80
- options.add_argument(' --disable-web-security') # allow cross origin
81
 
82
  driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
83
 
@@ -87,33 +66,23 @@ def _start_browser():
87
  app.logger.info(USER_AGENT)
88
 
89
  driver.quit()
90
- #restart with user agent
91
  options = uc.ChromeOptions()
92
  options.add_argument('--no-sandbox')
93
  options.add_argument('--window-size=1920,1080')
94
- # todo: this param shows a warning in chrome head-full
95
  options.add_argument('--disable-setuid-sandbox')
96
  options.add_argument('--disable-dev-shm-usage')
97
- # this option removes the zygote sandbox (it seems that the resolution is a bit faster)
98
  options.add_argument('--no-zygote')
99
- # attempt to fix Docker ARM32 build
100
  options.add_argument('--disable-gpu-sandbox')
101
  options.add_argument('--disable-software-rasterizer')
102
  options.add_argument('--ignore-certificate-errors')
103
  options.add_argument('--ignore-ssl-errors')
104
- # fix GL errors in ASUSTOR NAS
105
- # https://github.com/FlareSolverr/FlareSolverr/issues/782
106
- # https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
107
- # https://peter.sh/experiments/chromium-command-line-switches/#use-gl
108
  options.add_argument('--use-gl=swiftshader')
109
  if USER_AGENT is not None:
110
  options.add_argument('--user-agent=%s' % USER_AGENT)
111
  language = os.environ.get('LANG', None)
112
  if language is not None:
113
  options.add_argument('--lang=%s' % language)
114
-
115
- # added by me
116
- options.add_argument(' --disable-web-security') # allow cross origin
117
 
118
  driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
119
 
@@ -121,7 +90,6 @@ def _start_browser():
121
 
122
  app.logger.info("browser started")
123
 
124
-
125
  def _stop_browser():
126
  global driver
127
  driver.quit()
@@ -132,7 +100,7 @@ def _reset_stop_timer():
132
  global timer
133
  if timer is not None:
134
  timer.cancel()
135
- timer = Timer(5*60, _stop_browser) # _stop_browser executed in another thread...
136
  timer.start()
137
 
138
  @app.route("/")
@@ -160,12 +128,25 @@ def text():
160
  url = request.args.get('url', '')
161
  driver.get(url)
162
  text = driver.page_source
163
- #driver.close()
164
  return text
165
 
166
  @app.route("/screenshot")
167
  def screenshot():
168
- return "todo"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
 
170
  @app.route("/evaluate")
171
  def evaluate():
@@ -181,11 +162,10 @@ def fetch():
181
 
182
  url = request.args.get('url', '')
183
 
184
- #driver.get('https://example.com')
185
  driver.get('https://i-invdn-com.investing.com/redesign/images/seo/investing_300X300.png')
186
 
187
  script = """
188
- var callback = arguments[arguments.length - 1]; // this is the callback to call when you are done
189
  (async function(){
190
  try {
191
  let res = await fetch('%s', {headers:{'domain-id':'www'}});
 
1
+ from flask import Flask, request, send_file
 
2
  import undetected_chromedriver as uc
3
  import re
4
  import os
5
  from threading import Timer
6
+ import time
 
 
 
 
 
 
 
7
 
8
  driver = None
9
  XVFB_DISPLAY = None
 
24
  global CHROME_EXE_PATH
25
  if CHROME_EXE_PATH is not None:
26
  return CHROME_EXE_PATH
 
27
  chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
28
  if os.path.exists(chrome_path):
29
  CHROME_EXE_PATH = chrome_path
30
  return CHROME_EXE_PATH
 
31
  CHROME_EXE_PATH = uc.find_chrome_executable()
32
  return CHROME_EXE_PATH
33
 
 
40
 
41
  _start_xvfb_display()
42
 
 
43
  options = uc.ChromeOptions()
44
  options.add_argument('--no-sandbox')
45
  options.add_argument('--window-size=1920,1080')
 
46
  options.add_argument('--disable-setuid-sandbox')
47
  options.add_argument('--disable-dev-shm-usage')
 
48
  options.add_argument('--no-zygote')
 
49
  options.add_argument('--disable-gpu-sandbox')
50
  options.add_argument('--disable-software-rasterizer')
51
  options.add_argument('--ignore-certificate-errors')
52
  options.add_argument('--ignore-ssl-errors')
 
 
 
 
53
  options.add_argument('--use-gl=swiftshader')
 
54
  if USER_AGENT is not None:
55
  options.add_argument('--user-agent=%s' % USER_AGENT)
56
  language = os.environ.get('LANG', None)
57
  if language is not None:
58
  options.add_argument('--lang=%s' % language)
59
+ options.add_argument(' --disable-web-security')
 
 
60
 
61
  driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
62
 
 
66
  app.logger.info(USER_AGENT)
67
 
68
  driver.quit()
 
69
  options = uc.ChromeOptions()
70
  options.add_argument('--no-sandbox')
71
  options.add_argument('--window-size=1920,1080')
 
72
  options.add_argument('--disable-setuid-sandbox')
73
  options.add_argument('--disable-dev-shm-usage')
 
74
  options.add_argument('--no-zygote')
 
75
  options.add_argument('--disable-gpu-sandbox')
76
  options.add_argument('--disable-software-rasterizer')
77
  options.add_argument('--ignore-certificate-errors')
78
  options.add_argument('--ignore-ssl-errors')
 
 
 
 
79
  options.add_argument('--use-gl=swiftshader')
80
  if USER_AGENT is not None:
81
  options.add_argument('--user-agent=%s' % USER_AGENT)
82
  language = os.environ.get('LANG', None)
83
  if language is not None:
84
  options.add_argument('--lang=%s' % language)
85
+ options.add_argument(' --disable-web-security')
 
 
86
 
87
  driver = uc.Chrome(options=options, headless=False, version_main=None, driver_executable_path="/app/chromedriver", browser_executable_path=get_chrome_exe_path())
88
 
 
90
 
91
  app.logger.info("browser started")
92
 
 
93
  def _stop_browser():
94
  global driver
95
  driver.quit()
 
100
  global timer
101
  if timer is not None:
102
  timer.cancel()
103
+ timer = Timer(5*60, _stop_browser)
104
  timer.start()
105
 
106
  @app.route("/")
 
128
  url = request.args.get('url', '')
129
  driver.get(url)
130
  text = driver.page_source
 
131
  return text
132
 
133
  @app.route("/screenshot")
134
  def screenshot():
135
+ global driver
136
+ if driver is None:
137
+ _start_browser()
138
+ else:
139
+ _reset_stop_timer()
140
+
141
+ url = request.args.get('url', 'https://www.youtube.com')
142
+ driver.get(url)
143
+
144
+ time.sleep(5) # Adjust the sleep time as needed
145
+
146
+ screenshot_path = "/tmp/screenshot.png"
147
+ driver.save_screenshot(screenshot_path)
148
+
149
+ return send_file(screenshot_path, mimetype='image/png')
150
 
151
  @app.route("/evaluate")
152
  def evaluate():
 
162
 
163
  url = request.args.get('url', '')
164
 
 
165
  driver.get('https://i-invdn-com.investing.com/redesign/images/seo/investing_300X300.png')
166
 
167
  script = """
168
+ var callback = arguments[arguments.length - 1];
169
  (async function(){
170
  try {
171
  let res = await fetch('%s', {headers:{'domain-id':'www'}});