Update tariff_scraper.py

tariff_scraper.py

@@ -1,20 +1,22 @@
 import requests
 from bs4 import BeautifulSoup
+import pandas as pd
 
 # Define the URL for PESCO tariff rates
 TARIFF_URLS = {
     "PESCO": "https://onlinepescobill.pk/pesco-tariff-rates/"
 }
 
-def scrape_tariff_data(url):
+def scrape_tariff_data_to_csv(url, output_file="pesco_tariff_data.csv"):
     """
-    Scrape tariff data from the given URL.
+    Scrape tariff data from the given URL and save it to a CSV file.
 
     Args:
         url (str): The URL of the tariff page to scrape.
+        output_file (str): The name of the CSV file to save data.
 
     Returns:
-        list: A list of strings representing the rows of tariff data.
+        str: The name of the CSV file if successful, or an error message.
     """
     try:
         # Send an HTTP GET request to the specified URL
@@ -27,31 +29,33 @@ def scrape_tariff_data(url):
         # Extract table rows
         tariff_table = soup.find('table')
         if not tariff_table:
-            return ["No table found on the webpage."]
+            return "No table found on the webpage."
 
+        # Extract data and convert it into a structured format
         data = []
         table_rows = tariff_table.find_all('tr')
         for row in table_rows:
-            # Extract text from each <td> or <th> within the row
-            row_text = ' | '.join(
-                col.get_text(strip=True) for col in row.find_all(['th', 'td'])
-            )
-            if row_text:  # Add only rows that have meaningful data
-                data.append(row_text)
-
-        return data if data else ["No data found in the table."]
+            cols = [col.get_text(strip=True) for col in row.find_all(['th', 'td'])]
+            data.append(cols)
+
+        # Save the data to a CSV file
+        df = pd.DataFrame(data[1:], columns=data[0])  # Use the first row as headers
+        df.to_csv(output_file, index=False)
+
+        return output_file
     except requests.exceptions.RequestException as e:
         # Handle request errors (e.g., connection issues, timeout)
-        return [f"Request error: {e}"]
+        return f"Request error: {e}"
     except Exception as e:
         # Handle other potential errors
-        return [f"An unexpected error occurred: {e}"]
+        return f"An unexpected error occurred: {e}"
 
 if __name__ == "__main__":
-    # Test the scraper
+    # Test the scraper and save data to a CSV file
     url = TARIFF_URLS["PESCO"]
-    print(f"Fetching tariff data from {url}...\n")
-    tariff_data = scrape_tariff_data(url)
-    print("Tariff Data:")
-    for row in tariff_data:
-        print(row)
+    output_file = "pesco_tariff_data.csv"
+    result = scrape_tariff_data_to_csv(url, output_file)
+    if result.endswith(".csv"):
+        print(f"Data successfully saved to {output_file}")
+    else:
+        print(f"Error: {result}")