Rename tariff_scraper.py to scraper.py

scraper.py

@@ -0,0 +1,69 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import time
+from random import randint
+
+def scrape_tariffs(urls):
+    data = []
+    for url in urls:
+        try:
+            response = requests.get(url, timeout=10)  # Added timeout
+            response.raise_for_status()  # Raise exception for bad status codes (4xx, 5xx)
+
+            # Scrape data if the response is OK
+            if response.status_code == 200:
+                soup = BeautifulSoup(response.content, "html.parser")
+                rows = soup.find_all("tr")
+
+                for row in rows:
+                    cells = row.find_all("td")
+                    if len(cells) >= 2:
+                        try:
+                            data.append({
+                                "category": cells[0].text.strip(),
+                                "rate": float(cells[1].text.strip().replace(",", ""))
+                            })
+                        except ValueError:
+                            continue
+
+        except requests.exceptions.RequestException as e:
+            print(f"Error fetching data from {url}: {e}")
+            print("Retrying...")
+
+            # Retry logic in case of failure (max 3 retries with random delay)
+            retries = 3
+            while retries > 0:
+                time.sleep(randint(1, 3))  # Sleep for a random time before retrying
+                retries -= 1
+                try:
+                    response = requests.get(url, timeout=10)
+                    response.raise_for_status()
+                    if response.status_code == 200:
+                        soup = BeautifulSoup(response.content, "html.parser")
+                        rows = soup.find_all("tr")
+
+                        for row in rows:
+                            cells = row.find_all("td")
+                            if len(cells) >= 2:
+                                try:
+                                    data.append({
+                                        "category": cells[0].text.strip(),
+                                        "rate": float(cells[1].text.strip().replace(",", ""))
+                                    })
+                                except ValueError:
+                                    continue
+                        break
+                except requests.exceptions.RequestException:
+                    print(f"Retry failed: {e}")
+                    continue
+
+        # Sleep between requests to avoid hitting the servers too quickly
+        time.sleep(randint(2, 5))
+
+    if data:
+        df = pd.DataFrame(data)
+        df.to_csv("data/tariffs.csv", index=False)
+        print("Tariff data saved successfully.")
+    else:
+        print("No tariff data found.")

tariff_scraper.py

@@ -1,48 +0,0 @@
-import requests
-from bs4 import BeautifulSoup
-import pandas as pd
-
-def fetch_tariff_data(url):
-    """Fetch tariff data from the provided URL."""
-    response = requests.get(url)
-    if response.status_code != 200:
-        raise Exception(f"Failed to fetch data from {url}, status code: {response.status_code}")
-
-    soup = BeautifulSoup(response.content, 'html.parser')
-    sections = soup.find_all('tr', id='table_heading')
-
-    tariff_data = {}
-    for section in sections:
-        heading = section.find('td').get_text(strip=True)
-        rows = section.find_next_siblings('tr')
-
-        data = []
-        for row in rows:
-            columns = row.find_all('td')
-            if len(columns) >= 5:  # Ensure it has the expected number of columns
-                data.append({
-                    "Sr. No.": columns[0].get_text(strip=True),
-                    "Category": columns[1].get_text(strip=True),
-                    "Fixed Rs/Cons/M": columns[2].get_text(strip=True),
-                    "Fixed Rs/kW/M": columns[3].get_text(strip=True),
-                    "Variable Rs/kWh": columns[4].get_text(strip=True),
-                })
-
-        tariff_data[heading] = pd.DataFrame(data)
-
-    return tariff_data
-
-def save_tariff_data(tariff_data, file_path):
-    """Save the fetched tariff data to an Excel file."""
-    with pd.ExcelWriter(file_path) as writer:
-        for heading, df in tariff_data.items():
-            df.to_excel(writer, sheet_name=heading[:31], index=False)
-
-if __name__ == "__main__":
-    url = "https://iesco.com.pk/index.php/customer-services/tariff-guide"
-    try:
-        tariff_data = fetch_tariff_data(url)
-        save_tariff_data(tariff_data, "tariff_data.xlsx")
-        print("Tariff data successfully saved to tariff_data.xlsx")
-    except Exception as e:
-        print(f"Error: {e}")