Update app.py

app.py

@@ -2,45 +2,50 @@ import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 
-def scrape_tariff_data(url):
+def scrape_and_clean_tariff_data(url):
     try:
         response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
-        response.raise_for_status()  # Raise an error for bad responses
+        response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
 
         # Extract specific elements based on the webpage structure
-        # Assume tariff data is in <table> tags
         tariff_sections = soup.find_all('table')
 
-        data = []
+        cleaned_data = []
         for section in tariff_sections:
             table_rows = section.find_all('tr')
             for row in table_rows:
-                row_text = ' | '.join(
-                    col.get_text(strip=True) for col in row.find_all(['th', 'td'])
-                )
-                if row_text:  # Add the row text only if it contains data
-                    data.append(row_text)
+                # Extract text for each cell and handle missing/empty values
+                row_data = [col.get_text(strip=True) if col.get_text(strip=True) else "N/A" 
+                            for col in row.find_all(['th', 'td'])]
+                
+                # Skip rows that are completely empty
+                if any(cell != "N/A" for cell in row_data):
+                    cleaned_data.append(row_data)
 
-        return data  # Returns a list of row strings
+        return cleaned_data  # Returns a list of cleaned rows
     except Exception as e:
         return f"An error occurred: {e}"
 
 def main():
-    st.title("Electricity Tariff Scraper")
+    st.title("Electricity Tariff Scraper with Data Cleaning")
     st.write("Enter the URL of the electricity tariff page:")
 
     url = st.text_input("URL", "https://iesco.com.pk/index.php/customer-services/tariff-guide")
     
-    if st.button("Scrape"):
+    if st.button("Scrape and Clean"):
         if url:
             with st.spinner("Scraping data..."):
-                data = scrape_tariff_data(url)
+                data = scrape_and_clean_tariff_data(url)
                 if isinstance(data, list):
-                    st.success("Data scraped successfully!")
-                    st.write("Here is a preview of the data:")
-                    for row in data[:10]:  # Show only the first 10 rows for readability
-                        st.write(row)
+                    st.success("Data scraped and cleaned successfully!")
+                    st.write("Here is a preview of the cleaned data:")
+                    
+                    # Show cleaned data in a readable format
+                    for row in data[:10]:  # Show first 10 rows
+                        st.write(" | ".join(row))
+                    
+                    st.write(f"Total rows cleaned: {len(data)}")
                 else:
                     st.error(data)
         else: