Rename app.py to tariff_scraper.py

app.py → tariff_scraper.py

@@ -2,50 +2,45 @@ import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 
-def scrape_and_clean_tariff_data(url):
+def scrape_tariff_data(url):
     try:
         response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
-        response.raise_for_status()
+        response.raise_for_status()  # Raise an error for bad responses
         soup = BeautifulSoup(response.text, 'html.parser')
 
         # Extract specific elements based on the webpage structure
+        # Assume tariff data is in <table> tags
         tariff_sections = soup.find_all('table')
 
-        cleaned_data = []
+        data = []
         for section in tariff_sections:
             table_rows = section.find_all('tr')
             for row in table_rows:
-                # Extract text for each cell and handle missing/empty values
-                row_data = [col.get_text(strip=True) if col.get_text(strip=True) else "N/A" 
-                            for col in row.find_all(['th', 'td'])]
-                
-                # Skip rows that are completely empty
-                if any(cell != "N/A" for cell in row_data):
-                    cleaned_data.append(row_data)
+                row_text = ' | '.join(
+                    col.get_text(strip=True) for col in row.find_all(['th', 'td'])
+                )
+                if row_text:  # Add the row text only if it contains data
+                    data.append(row_text)
 
-        return cleaned_data  # Returns a list of cleaned rows
+        return data  # Returns a list of row strings
     except Exception as e:
         return f"An error occurred: {e}"
 
 def main():
-    st.title("Electricity Tariff Scraper with Data Cleaning")
+    st.title("Electricity Tariff Scraper")
     st.write("Enter the URL of the electricity tariff page:")
 
     url = st.text_input("URL", "https://iesco.com.pk/index.php/customer-services/tariff-guide")
     
-    if st.button("Scrape and Clean"):
+    if st.button("Scrape"):
         if url:
             with st.spinner("Scraping data..."):
-                data = scrape_and_clean_tariff_data(url)
+                data = scrape_tariff_data(url)
                 if isinstance(data, list):
-                    st.success("Data scraped and cleaned successfully!")
-                    st.write("Here is a preview of the cleaned data:")
-                    
-                    # Show cleaned data in a readable format
-                    for row in data[:10]:  # Show first 10 rows
-                        st.write(" | ".join(row))
-                    
-                    st.write(f"Total rows cleaned: {len(data)}")
+                    st.success("Data scraped successfully!")
+                    st.write("Here is a preview of the data:")
+                    for row in data[:10]:  # Show only the first 10 rows for readability
+                        st.write(row)
                 else:
                     st.error(data)
         else: