Update app.py

app.py

@@ -3,6 +3,14 @@ from bs4 import BeautifulSoup
 from urllib.parse import urlencode
 import requests
 from fastmcp import FastMCP
+import logging
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+import time
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
 
 # Initialize FastMCP server
 mcp = FastMCP("Canada Job Bank Scraper Agent")
@@ -41,20 +49,36 @@ def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = N
     }
 
     try:
+        # Try requests first
+        logger.info(f"Attempting to scrape: {url}")
         response = requests.get(url, headers=headers, timeout=10)
         response.raise_for_status()
-
         soup = BeautifulSoup(response.text, "html.parser")
-        cards = soup.find_all("article", class_="resultJobItem")
+        cards = soup.find_all("article", class_="job-result")  # Updated class name (verify)
+
+        if not cards:
+            logger.warning("No job cards found with requests. Trying Selenium...")
+            # Fallback to Selenium for dynamic content
+            chrome_options = Options()
+            chrome_options.add_argument("--headless")
+            chrome_options.add_argument("--no-sandbox")
+            chrome_options.add_argument("--disable-dev-shm-usage")
+            driver = webdriver.Chrome(options=chrome_options)
+            driver.get(url)
+            time.sleep(3)  # Wait for JavaScript to load
+            soup = BeautifulSoup(driver.page_source, "html.parser")
+            driver.quit()
+            cards = soup.find_all("article", class_="job-result")
 
         if not cards:
+            logger.error("No job listings found. Possible website structure change.")
             return {"error": "No job listings found or website structure changed."}
 
         jobs = []
         for card in cards[:limit]:
-            title_elem = card.find("span", class_="noctitle")
-            company_elem = card.find("li", class_="business")
-            location_elem = card.find("li", class_="location")
+            title_elem = card.find("span", class_="job-title")  # Updated class
+            company_elem = card.find("li", class_="employer")   # Updated class
+            location_elem = card.find("li", class_="job-location")  # Updated class
             link_elem = card.find("a", href=True)
 
             link = link_elem.get("href") if link_elem else None
@@ -69,15 +93,20 @@ def search_jobs_tool(query: str, location: str, limit: int = 10, salary: str = N
             }
             jobs.append(job)
 
+        logger.info(f"Found {len(jobs)} job listings.")
         return {"jobs": jobs}
 
     except requests.exceptions.HTTPError as http_err:
+        logger.error(f"HTTP error: {http_err}")
         return {"error": f"HTTP error occurred: {http_err}"}
     except requests.exceptions.Timeout:
+        logger.error("Request timed out.")
         return {"error": "Request timed out. Please try again later."}
     except requests.exceptions.RequestException as req_err:
+        logger.error(f"Request error: {req_err}")
         return {"error": f"Request error: {req_err}"}
     except Exception as e:
+        logger.error(f"Unexpected error: {str(e)}")
         return {"error": f"Unexpected error: {str(e)}"}
 
 def search_jobs_ui(query, location, limit=10, salary=None, job_type=None):
@@ -125,4 +154,4 @@ app = gr.Interface(
 )
 
 if __name__ == "__main__":
-    app.launch(mcp_server=True)
+    app.launch(mcp_server=True, server_name="0.0.0.0", server_port=7860)