Update app.py

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 from bs4 import BeautifulSoup
 from urllib.parse import urlencode
-import requests
+import cloudscraper
 from fastmcp import FastMCP
 import logging
 
@@ -15,7 +15,7 @@ mcp = FastMCP("Canada Job Bank Scraper Agent")
 @mcp.tool(name="search_jobs")
 def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = None, job_type: str = None) -> dict:
     """
-    Scrape job listings from the Canada Job Bank website using requests only.
+    Scrape job listings from the Canada Job Bank website using cloudscraper.
 
     Args:
         query (str): Job title or keyword to search for.
@@ -49,14 +49,15 @@ def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = N
 
     try:
         logger.info(f"Attempting to scrape: {url}")
-        response = requests.get(url, headers=headers, timeout=10)
+        scraper = cloudscraper.create_scraper()
+        response = scraper.get(url, headers=headers, timeout=10)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, "html.parser")
         cards = soup.find_all("article", class_="job-result")  # Verify class name
 
         if not cards:
             logger.warning("No job cards found. The website may use JavaScript or the HTML structure may have changed.")
-            logger.debug(f"HTML sample: {soup.prettify()[:1000]}")
+            logger.debug(f"HTML sample: {soup.prettify()[:2000]}")  # Increased sample size for debugging
             return {"error": "No job listings found. The website may use JavaScript or the HTML structure may have changed."}
 
         jobs = []
@@ -81,6 +82,9 @@ def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = N
         logger.info(f"Found {len(jobs)} job listings.")
         return {"jobs": jobs}
 
+    except cloudscraper.exceptions.CloudflareChallengeError as cf_err:
+        logger.error(f"Cloudflare challenge error: {cf_err}")
+        return {"error": f"Cloudflare challenge error: {cf_err}"}
     except requests.exceptions.HTTPError as http_err:
         logger.error(f"HTTP error: {http_err}")
         return {"error": f"HTTP error occurred: {http_err}"}
@@ -134,7 +138,7 @@ app = gr.Interface(
     ],
     outputs=gr.Markdown(),
     title="Canada Job Bank Job Search",
-    description="Search jobs by scraping Canada Job Bank using FastMCP and Gradio.",
+    description="Search jobs by scraping Canada Job Bank using FastMCP and cloudscraper.",
     theme="huggingface"
 )