Update app.py

app.py

@@ -1,27 +1,16 @@
 import gradio as gr
-import cloudscraper
 from bs4 import BeautifulSoup
 from urllib.parse import urlencode
 from fastmcp import FastMCP
+import undetected_chromedriver as uc
+from selenium.webdriver.chrome.options import Options
+import time
 
 # Initialize FastMCP agent
 mcp = FastMCP("Indeed Web Scraper Agent")
 
 @mcp.tool(name="search_jobs")
 def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = None, job_type: str = None):
-    """
-    Scrape jobs from Indeed website using cloudscraper + BeautifulSoup.
-
-    Args:
-        query (str): Job title or keywords to search for.
-        location (str): Location (city, region) to search jobs in.
-        limit (int): Number of job results to return (default 10).
-        salary (str, optional): Not used in scraper, reserved for future.
-        job_type (str, optional): Not used in scraper, reserved for future.
-
-    Returns:
-        dict: Contains a list of jobs with title, company, location, and url.
-    """
     base_url = "https://ca.indeed.com/jobs?"
 
     params = {
@@ -29,23 +18,25 @@ def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = N
         "l": location,
         "sort": "date",
     }
-
     url = base_url + urlencode(params)
-    headers = {
-        "User-Agent": (
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/114.0.0.0 Safari/537.36"
-        ),
-        "Accept-Language": "en-US,en;q=0.9",
-    }
+
+    options = Options()
+    options.headless = True  # 如要睇實際畫面，改 False
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+    options.add_argument("--disable-blink-features=AutomationControlled")
+    options.add_argument("--disable-gpu")
 
     try:
-        scraper = cloudscraper.create_scraper()
-        response = scraper.get(url, headers=headers, timeout=15)
-        response.raise_for_status()
+        # 用 undetected_chromedriver 產生一個 selenium ChromeDriver
+        driver = uc.Chrome(options=options)
+        driver.get(url)
+        time.sleep(5)  # 等頁面載入好
 
-        soup = BeautifulSoup(response.text, "html.parser")
+        html = driver.page_source
+        driver.quit()
+
+        soup = BeautifulSoup(html, "html.parser")
 
         jobs = []
         cards = soup.find_all("a", class_="tapItem")
@@ -70,7 +61,6 @@ def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = N
     except Exception as e:
         return {"error": str(e)}
 
-
 def search_jobs_ui(query, location, limit=10, salary=None, job_type=None):
     result = search_jobs_tool(query, location, limit, salary, job_type)
 
@@ -99,8 +89,8 @@ app = gr.Interface(
         gr.Textbox(label="Job Type (optional, ignored)")
     ],
     outputs="markdown",
-    title="Indeed Job Search (with cloudscraper) + FastMCP",
-    description="Search jobs by scraping Indeed.ca using cloudscraper and BeautifulSoup."
+    title="Indeed Job Search (with Selenium) + FastMCP",
+    description="Search jobs by scraping Indeed.ca using Selenium and BeautifulSoup."
 )
 
 if __name__ == "__main__":