Update tariff_scraper.py

tariff_scraper.py

@@ -2,29 +2,35 @@ import requests
 from bs4 import BeautifulSoup
 
 def scrape_tariff_data(url):
+    """
+    Scrape tariff data from the provided URL.
+    """
     try:
         # Fetch the webpage
         response = requests.get(url)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
 
-        # Find the table containing tariff data
-        table = soup.find("table")  # Adjust this if the table has specific attributes like class or id
-        if not table:
-            return "Error: Could not find the tariff table on the page."
+        # Find all rows in the table with the specific structure
+        rows = soup.find_all("tr", id=lambda x: x and x.startswith("table_"))  # Matches rows like "table_even_row"
 
-        # Extract table rows
-        rows = table.find_all("tr")
         if not rows:
             return "Error: No rows found in the tariff table."
 
-        # Parse table rows and extract data
+        # Parse the rows
         tariff_data = []
         for row in rows:
-            cells = row.find_all(["td", "th"])  # Include both header and data cells
-            tariff_data.append([cell.get_text(strip=True) for cell in cells])
+            # Extract all cell values (text inside <td>)
+            cells = row.find_all("td")
+            row_data = [cell.get_text(strip=True) for cell in cells]
+            
+            # Only include rows with meaningful data (exclude rows with all `-`)
+            if any(cell != '-' for cell in row_data):
+                tariff_data.append(row_data)
+
+        if not tariff_data:
+            return "Error: No meaningful data found in the tariff table."
 
-        # Return the tariff data as a list of lists (rows)
         return tariff_data
 
     except Exception as e: