Update tariff_scraper.py

tariff_scraper.py

@@ -1,33 +1,48 @@
 import requests
 from bs4 import BeautifulSoup
+import pandas as pd
 
-def scrape_tariff_data(url):
+def fetch_tariff_data(url):
+    """Fetch tariff data from the provided URL."""
+    response = requests.get(url)
+    if response.status_code != 200:
+        raise Exception(f"Failed to fetch data from {url}, status code: {response.status_code}")
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    sections = soup.find_all('tr', id='table_heading')
+
+    tariff_data = {}
+    for section in sections:
+        heading = section.find('td').get_text(strip=True)
+        rows = section.find_next_siblings('tr')
+
+        data = []
+        for row in rows:
+            columns = row.find_all('td')
+            if len(columns) >= 5:  # Ensure it has the expected number of columns
+                data.append({
+                    "Sr. No.": columns[0].get_text(strip=True),
+                    "Category": columns[1].get_text(strip=True),
+                    "Fixed Rs/Cons/M": columns[2].get_text(strip=True),
+                    "Fixed Rs/kW/M": columns[3].get_text(strip=True),
+                    "Variable Rs/kWh": columns[4].get_text(strip=True),
+                })
+
+        tariff_data[heading] = pd.DataFrame(data)
+
+    return tariff_data
+
+def save_tariff_data(tariff_data, file_path):
+    """Save the fetched tariff data to an Excel file."""
+    with pd.ExcelWriter(file_path) as writer:
+        for heading, df in tariff_data.items():
+            df.to_excel(writer, sheet_name=heading[:31], index=False)
+
+if __name__ == "__main__":
+    url = "https://iesco.com.pk/index.php/customer-services/tariff-guide"
     try:
-        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
-        response.raise_for_status()  # Raise an error for bad responses
-        soup = BeautifulSoup(response.text, 'html.parser')
-
-        # Extract the table containing tariff data
-        tariff_sections = soup.find_all('table')
-        if not tariff_sections:
-            return "Error: No tables found on the webpage."
-
-        tariff_data = {}
-        for section in tariff_sections:
-            rows = section.find_all('tr')  # Find all rows in the table
-            for row in rows:
-                columns = row.find_all('td')  # Extract all table data (td) columns
-                if len(columns) >= 5:  # Check if the row has enough columns
-                    category = columns[1].get_text(strip=True)  # Tariff category
-                    rate = columns[4].get_text(strip=True)      # Variable charges (Rs./kWh)
-
-                    # Add to the dictionary if the rate is numeric
-                    if rate.replace('.', '', 1).isdigit():
-                        tariff_data[category] = float(rate)
-
-        if not tariff_data:
-            return "Error: No valid tariff rates found."
-
-        return tariff_data  # Return structured tariff data as a dictionary
+        tariff_data = fetch_tariff_data(url)
+        save_tariff_data(tariff_data, "tariff_data.xlsx")
+        print("Tariff data successfully saved to tariff_data.xlsx")
     except Exception as e:
-        return f"An error occurred: {e}"
+        print(f"Error: {e}")