plan291037 commited on
Commit
a8db444
·
verified ·
1 Parent(s): fa551ee

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +18 -17
app.py CHANGED
@@ -1,11 +1,14 @@
1
  import os, asyncio
2
  from fastapi import FastAPI, HTTPException, Header, Response
3
  from pydantic import BaseModel, HttpUrl
4
- from seleniumbase import Driver
 
 
5
 
6
  API_KEY = os.getenv("OPENER_API_KEY", "")
7
- PROFILE = os.getenv("SB_PROFILE_DIR", "/tmp/sb-profile")
8
  UA = os.getenv("UA", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36")
 
9
 
10
  app = FastAPI()
11
  drv = None
@@ -15,33 +18,31 @@ worker = None
15
  class OpenPayload(BaseModel):
16
  url: HttpUrl
17
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  async def _worker():
19
  while True:
20
  url = await queue.get()
21
  try:
22
  await asyncio.to_thread(drv.get, url)
23
  print("[open] done", url)
24
- except Exception as e:
25
- print("[open] err", url, e)
26
  finally:
27
  queue.task_done()
28
 
29
  @app.on_event("startup")
30
  async def startup():
31
  global drv, queue, worker
32
- os.chdir("/tmp")
33
- os.makedirs("downloaded_files", exist_ok=True)
34
- os.makedirs(PROFILE, exist_ok=True)
35
- drv = Driver(
36
- uc=True,
37
- headless=True,
38
- incognito=False,
39
- user_data_dir=PROFILE,
40
- page_load_strategy="normal",
41
- agent=UA,
42
- no_sandbox=True,
43
- disable_gpu=True,
44
- )
45
  drv.get("data:text/html,ready")
46
  queue = asyncio.Queue()
47
  worker = asyncio.create_task(_worker())
 
1
  import os, asyncio
2
  from fastapi import FastAPI, HTTPException, Header, Response
3
  from pydantic import BaseModel, HttpUrl
4
+ from selenium import webdriver
5
+ from selenium.webdriver.chrome.service import Service
6
+ from selenium.webdriver.chrome.options import Options
7
 
8
  API_KEY = os.getenv("OPENER_API_KEY", "")
9
+ PROFILE = os.getenv("CHROME_PROFILE_DIR", "/tmp/chrome-profile")
10
  UA = os.getenv("UA", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36")
11
+ CHROMEDRIVER = os.getenv("CHROMEDRIVER", "/usr/bin/chromedriver") # debian usually symlinks here
12
 
13
  app = FastAPI()
14
  drv = None
 
18
  class OpenPayload(BaseModel):
19
  url: HttpUrl
20
 
21
+ def _build_driver():
22
+ os.makedirs(PROFILE, exist_ok=True)
23
+ opts = Options()
24
+ opts.add_argument("--headless=new")
25
+ opts.add_argument("--no-sandbox")
26
+ opts.add_argument("--disable-gpu")
27
+ opts.add_argument("--disable-dev-shm-usage")
28
+ opts.add_argument(f"--user-agent={UA}")
29
+ opts.add_argument(f"--user-data-dir={PROFILE}")
30
+ svc = Service(executable_path=CHROMEDRIVER)
31
+ return webdriver.Chrome(service=svc, options=opts)
32
+
33
  async def _worker():
34
  while True:
35
  url = await queue.get()
36
  try:
37
  await asyncio.to_thread(drv.get, url)
38
  print("[open] done", url)
 
 
39
  finally:
40
  queue.task_done()
41
 
42
  @app.on_event("startup")
43
  async def startup():
44
  global drv, queue, worker
45
+ drv = _build_driver()
 
 
 
 
 
 
 
 
 
 
 
 
46
  drv.get("data:text/html,ready")
47
  queue = asyncio.Queue()
48
  worker = asyncio.create_task(_worker())