Update tariff_scraper.py

tariff_scraper.py

@@ -1,37 +1,50 @@
+import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 
 def scrape_tariff_data(url):
-    """
-    Scrape tariff data from the provided URL.
-    """
     try:
-        # Fetch the webpage
-        response = requests.get(url)
-        response.raise_for_status()
+        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
+        response.raise_for_status()  # Raise an error for bad responses
         soup = BeautifulSoup(response.text, 'html.parser')
 
-        # Find all rows in the table with the specific structure
-        rows = soup.find_all("tr", id=lambda x: x and x.startswith("table_"))  # Matches rows like "table_even_row"
+        # Extract specific elements based on the webpage structure
+        # Assume tariff data is in <table> tags
+        tariff_sections = soup.find_all('table')
 
-        if not rows:
-            return "Error: No rows found in the tariff table."
+        data = []
+        for section in tariff_sections:
+            table_rows = section.find_all('tr')
+            for row in table_rows:
+                row_text = ' | '.join(
+                    col.get_text(strip=True) for col in row.find_all(['th', 'td'])
+                )
+                if row_text:  # Add the row text only if it contains data
+                    data.append(row_text)
 
-        # Parse the rows
-        tariff_data = []
-        for row in rows:
-            # Extract all cell values (text inside <td>)
-            cells = row.find_all("td")
-            row_data = [cell.get_text(strip=True) for cell in cells]
-            
-            # Only include rows with meaningful data (exclude rows with all `-`)
-            if any(cell != '-' for cell in row_data):
-                tariff_data.append(row_data)
+        return data  # Returns a list of row strings
+    except Exception as e:
+        return f"An error occurred: {e}"
 
-        if not tariff_data:
-            return "Error: No meaningful data found in the tariff table."
+def main():
+    st.title("Electricity Tariff Scraper")
+    st.write("Enter the URL of the electricity tariff page:")
 
-        return tariff_data
+    url = st.text_input("URL", "https://iesco.com.pk/index.php/customer-services/tariff-guide")
+    
+    if st.button("Scrape"):
+        if url:
+            with st.spinner("Scraping data..."):
+                data = scrape_tariff_data(url)
+                if isinstance(data, list):
+                    st.success("Data scraped successfully!")
+                    st.write("Here is a preview of the data:")
+                    for row in data[:10]:  # Show only the first 10 rows for readability
+                        st.write(row)
+                else:
+                    st.error(data)
+        else:
+            st.error("Please enter a valid URL.")
 
-    except Exception as e:
-        return f"An error occurred while scraping: {e}"
+if __name__ == "__main__":
+    main()