usmanyousaf commited on
Commit
2e18161
·
verified ·
1 Parent(s): f560e5e

Update scrape.py

Browse files
Files changed (1) hide show
  1. scrape.py +3 -8
scrape.py CHANGED
@@ -2,25 +2,20 @@ from selenium import webdriver # type: ignore
2
  from selenium.webdriver.chrome.service import Service # type: ignore
3
  from selenium.webdriver.chrome.options import Options # type: ignore
4
  from bs4 import BeautifulSoup # type: ignore
5
- from dotenv import load_dotenv # type: ignore
6
  import os
7
  import time
8
 
9
- # Load environment variables
10
  load_dotenv()
11
 
12
- CHROME_DRIVER_PATH = os.getenv("CHROME_DRIVER_PATH", "./chrome")
13
 
14
  def scrape_website(website):
15
  print("Connecting to Chrome Browser...")
16
 
17
- # Setup ChromeDriver service and options for headless scraping
18
  service = Service(CHROME_DRIVER_PATH)
19
  options = Options()
20
- options.add_argument("--headless") # Run Chrome in headless mode
21
- options.add_argument("--no-sandbox")
22
- options.add_argument("--disable-dev-shm-usage")
23
-
24
  driver = webdriver.Chrome(service=service, options=options)
25
 
26
  try:
 
2
  from selenium.webdriver.chrome.service import Service # type: ignore
3
  from selenium.webdriver.chrome.options import Options # type: ignore
4
  from bs4 import BeautifulSoup # type: ignore
5
+ from dotenv import load_dotenv # type: ignore
6
  import os
7
  import time
8
 
 
9
  load_dotenv()
10
 
11
+ CHROME_DRIVER_PATH = os.getenv("./chrome")
12
 
13
  def scrape_website(website):
14
  print("Connecting to Chrome Browser...")
15
 
16
+ # Setup ChromeDriver service and options
17
  service = Service(CHROME_DRIVER_PATH)
18
  options = Options()
 
 
 
 
19
  driver = webdriver.Chrome(service=service, options=options)
20
 
21
  try: