Update scraper.py

scraper.py

@@ -1,26 +1,51 @@
-# scraper.py
 import requests
 from bs4 import BeautifulSoup
 
 def fetch_tariff_data(url):
+    # Send a GET request to fetch the raw HTML content
     response = requests.get(url)
+    
+    if response.status_code != 200:
+        print(f"Failed to retrieve data from {url}. Status code: {response.status_code}")
+        return None
+    
+    # Print the first 500 characters of the response content for debugging
+    print(f"Successfully fetched data from {url}. First 500 characters of HTML:")
+    print(response.text[:500])  # Print the first 500 characters of HTML to inspect
+    
     soup = BeautifulSoup(response.text, 'html.parser')
     tariff_data = parse_tariff_data(soup)
+    
+    # Return the parsed tariff data
     return tariff_data
 
 def parse_tariff_data(soup):
-    # Find the table that contains tariff data
+    # Try to find the table that contains tariff data
     table = soup.find('table')
+    
+    if not table:
+        print("No table found on this page.")
+        return []
+    
     tariff_data = []
     
-    for row in table.find_all('tr')[1:]:  # Skipping the header row
+    # Check if the table has rows and parse accordingly
+    rows = table.find_all('tr')[1:]  # Skipping the header row
+    if not rows:
+        print("No data rows found in the table.")
+        return []
+    
+    # Loop through each row and parse the tariff details
+    for row in rows:
         columns = row.find_all('td')
+        
         if len(columns) >= 2:  # Ensure there are at least two columns (slab and rate)
             slab = columns[0].text.strip()  # Slab details (e.g., 0-50 kWh)
             try:
-                # Attempt to clean and convert rate to a float
+                # Attempt to clean and convert the rate to a float
                 rate = float(columns[1].text.strip().replace(',', '').replace('Rs.', ''))
             except ValueError:
+                print(f"Skipping row with invalid rate: {columns[1].text.strip()}")
                 continue  # Skip rows that don’t have a valid rate
             
             # Handle the case where slab is not a valid numeric range
@@ -31,11 +56,13 @@ def parse_tariff_data(soup):
                     upper_limit = float(slab_range[1])
                     tariff_data.append({'slab': slab, 'rate': rate, 'lower_limit': lower_limit, 'upper_limit': upper_limit})
                 except ValueError:
+                    print(f"Skipping invalid slab range: {slab}")
                     continue  # Skip slabs that are not valid numeric ranges
             elif len(slab_range) == 1:
                 try:
                     lower_limit = float(slab_range[0])
                     tariff_data.append({'slab': slab, 'rate': rate, 'lower_limit': lower_limit, 'upper_limit': float('inf')})
                 except ValueError:
+                    print(f"Skipping invalid slab range: {slab}")
                     continue  # Skip slabs that are not valid numeric values
     return tariff_data