Update tariff_scraper.py

tariff_scraper.py

@@ -12,22 +12,30 @@ def fetch_tariff_data(url):
 
     tariff_data = {}
     for section in sections:
-        heading = section.find('td').get_text(strip=True)
-        rows = section.find_next_siblings('tr')
-
-        data = []
-        for row in rows:
-            columns = row.find_all('td')
-            if len(columns) >= 5:  # Ensure it has the expected number of columns
-                data.append({
+        # Get the section heading
+        heading_cell = section.find('td')
+        if not heading_cell:
+            continue
+        heading = heading_cell.get_text(strip=True)
+        
+        # Get rows for this section
+        rows = []
+        sibling = section.find_next_sibling()
+        while sibling and sibling.name == 'tr' and sibling.get('id') != 'table_heading':
+            columns = sibling.find_all('td')
+            if len(columns) >= 5:  # Ensure the expected column count
+                rows.append({
                     "Sr. No.": columns[0].get_text(strip=True),
                     "Category": columns[1].get_text(strip=True),
                     "Fixed Rs/Cons/M": columns[2].get_text(strip=True),
                     "Fixed Rs/kW/M": columns[3].get_text(strip=True),
                     "Variable Rs/kWh": columns[4].get_text(strip=True),
                 })
+            sibling = sibling.find_next_sibling()
 
-        tariff_data[heading] = pd.DataFrame(data)
+        # Only add to the data if rows were found
+        if rows:
+            tariff_data[heading] = pd.DataFrame(rows)
 
     return tariff_data