Update tariff_scraper.py

tariff_scraper.py

@@ -1,4 +1,3 @@
-import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 
@@ -8,43 +7,27 @@ def scrape_tariff_data(url):
         response.raise_for_status()  # Raise an error for bad responses
         soup = BeautifulSoup(response.text, 'html.parser')
 
-        # Extract specific elements based on the webpage structure
-        # Assume tariff data is in <table> tags
+        # Extract the table containing tariff data
         tariff_sections = soup.find_all('table')
+        if not tariff_sections:
+            return "Error: No tables found on the webpage."
 
-        data = []
+        tariff_data = {}
         for section in tariff_sections:
-            table_rows = section.find_all('tr')
-            for row in table_rows:
-                row_text = ' | '.join(
-                    col.get_text(strip=True) for col in row.find_all(['th', 'td'])
-                )
-                if row_text:  # Add the row text only if it contains data
-                    data.append(row_text)
+            rows = section.find_all('tr')  # Find all rows in the table
+            for row in rows:
+                columns = row.find_all('td')  # Extract all table data (td) columns
+                if len(columns) >= 5:  # Check if the row has enough columns
+                    category = columns[1].get_text(strip=True)  # Tariff category
+                    rate = columns[4].get_text(strip=True)      # Variable charges (Rs./kWh)
 
-        return data  # Returns a list of row strings
-    except Exception as e:
-        return f"An error occurred: {e}"
-
-def main():
-    st.title("Electricity Tariff Scraper")
-    st.write("Enter the URL of the electricity tariff page:")
+                    # Add to the dictionary if the rate is numeric
+                    if rate.replace('.', '', 1).isdigit():
+                        tariff_data[category] = float(rate)
 
-    url = st.text_input("URL", "https://iesco.com.pk/index.php/customer-services/tariff-guide")
-    
-    if st.button("Scrape"):
-        if url:
-            with st.spinner("Scraping data..."):
-                data = scrape_tariff_data(url)
-                if isinstance(data, list):
-                    st.success("Data scraped successfully!")
-                    st.write("Here is a preview of the data:")
-                    for row in data[:10]:  # Show only the first 10 rows for readability
-                        st.write(row)
-                else:
-                    st.error(data)
-        else:
-            st.error("Please enter a valid URL.")
+        if not tariff_data:
+            return "Error: No valid tariff rates found."
 
-if __name__ == "__main__":
-    main()
+        return tariff_data  # Return structured tariff data as a dictionary
+    except Exception as e:
+        return f"An error occurred: {e}"