Update tariff_scraper.py

tariff_scraper.py

@@ -12,30 +12,22 @@ def fetch_tariff_data(url):
 
     tariff_data = {}
     for section in sections:
-        # Get the section heading
-        heading_cell = section.find('td')
-        if not heading_cell:
-            continue
-        heading = heading_cell.get_text(strip=True)
-        
-        # Get rows for this section
-        rows = []
-        sibling = section.find_next_sibling()
-        while sibling and sibling.name == 'tr' and sibling.get('id') != 'table_heading':
-            columns = sibling.find_all('td')
-            if len(columns) >= 5:  # Ensure the expected column count
-                rows.append({
+        heading = section.find('td').get_text(strip=True)
+        rows = section.find_next_siblings('tr')
+
+        data = []
+        for row in rows:
+            columns = row.find_all('td')
+            if len(columns) >= 5:  # Ensure it has the expected number of columns
+                data.append({
                     "Sr. No.": columns[0].get_text(strip=True),
                     "Category": columns[1].get_text(strip=True),
                     "Fixed Rs/Cons/M": columns[2].get_text(strip=True),
                     "Fixed Rs/kW/M": columns[3].get_text(strip=True),
                     "Variable Rs/kWh": columns[4].get_text(strip=True),
                 })
-            sibling = sibling.find_next_sibling()
 
-        # Only add to the data if rows were found
-        if rows:
-            tariff_data[heading] = pd.DataFrame(rows)
+        tariff_data[heading] = pd.DataFrame(data)
 
     return tariff_data
 
@@ -52,3 +44,9 @@ if __name__ == "__main__":
         print("Tariff data successfully saved to tariff_data.xlsx")
     except Exception as e:
         print(f"Error: {e}")
+
+
+
+
+
+