Rename scraper.py to tariff_scraper.py

scraper.py

@@ -1,69 +0,0 @@
-import requests
-from bs4 import BeautifulSoup
-import pandas as pd
-import time
-from random import randint
-
-def scrape_tariffs(urls):
-    data = []
-    for url in urls:
-        try:
-            response = requests.get(url, timeout=10)  # Added timeout
-            response.raise_for_status()  # Raise exception for bad status codes (4xx, 5xx)
-
-            # Scrape data if the response is OK
-            if response.status_code == 200:
-                soup = BeautifulSoup(response.content, "html.parser")
-                rows = soup.find_all("tr")
-
-                for row in rows:
-                    cells = row.find_all("td")
-                    if len(cells) >= 2:
-                        try:
-                            data.append({
-                                "category": cells[0].text.strip(),
-                                "rate": float(cells[1].text.strip().replace(",", ""))
-                            })
-                        except ValueError:
-                            continue
-
-        except requests.exceptions.RequestException as e:
-            print(f"Error fetching data from {url}: {e}")
-            print("Retrying...")
-
-            # Retry logic in case of failure (max 3 retries with random delay)
-            retries = 3
-            while retries > 0:
-                time.sleep(randint(1, 3))  # Sleep for a random time before retrying
-                retries -= 1
-                try:
-                    response = requests.get(url, timeout=10)
-                    response.raise_for_status()
-                    if response.status_code == 200:
-                        soup = BeautifulSoup(response.content, "html.parser")
-                        rows = soup.find_all("tr")
-
-                        for row in rows:
-                            cells = row.find_all("td")
-                            if len(cells) >= 2:
-                                try:
-                                    data.append({
-                                        "category": cells[0].text.strip(),
-                                        "rate": float(cells[1].text.strip().replace(",", ""))
-                                    })
-                                except ValueError:
-                                    continue
-                        break
-                except requests.exceptions.RequestException:
-                    print(f"Retry failed: {e}")
-                    continue
-
-        # Sleep between requests to avoid hitting the servers too quickly
-        time.sleep(randint(2, 5))
-
-    if data:
-        df = pd.DataFrame(data)
-        df.to_csv("data/tariffs.csv", index=False)
-        print("Tariff data saved successfully.")
-    else:
-        print("No tariff data found.")

tariff_scraper.py

@@ -0,0 +1,57 @@
+import requests
+from bs4 import BeautifulSoup
+
+# Define the URL for PESCO tariff rates
+TARIFF_URLS = {
+    "PESCO": "https://onlinepescobill.pk/pesco-tariff-rates/"
+}
+
+def scrape_tariff_data(url):
+    """
+    Scrape tariff data from the given URL.
+
+    Args:
+        url (str): The URL of the tariff page to scrape.
+
+    Returns:
+        list: A list of strings representing the rows of tariff data.
+    """
+    try:
+        # Send an HTTP GET request to the specified URL
+        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
+        response.raise_for_status()  # Raise an error for HTTP issues
+
+        # Parse the webpage content using BeautifulSoup
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        # Extract table rows
+        tariff_table = soup.find('table')
+        if not tariff_table:
+            return ["No table found on the webpage."]
+
+        data = []
+        table_rows = tariff_table.find_all('tr')
+        for row in table_rows:
+            # Extract text from each <td> or <th> within the row
+            row_text = ' | '.join(
+                col.get_text(strip=True) for col in row.find_all(['th', 'td'])
+            )
+            if row_text:  # Add only rows that have meaningful data
+                data.append(row_text)
+
+        return data if data else ["No data found in the table."]
+    except requests.exceptions.RequestException as e:
+        # Handle request errors (e.g., connection issues, timeout)
+        return [f"Request error: {e}"]
+    except Exception as e:
+        # Handle other potential errors
+        return [f"An unexpected error occurred: {e}"]
+
+if __name__ == "__main__":
+    # Test the scraper
+    url = TARIFF_URLS["PESCO"]
+    print(f"Fetching tariff data from {url}...\n")
+    tariff_data = scrape_tariff_data(url)
+    print("Tariff Data:")
+    for row in tariff_data:
+        print(row)