Engineer786 commited on
Commit
09b0dbf
·
verified ·
1 Parent(s): f8968ac

Update tariff_scraper.py

Browse files
Files changed (1) hide show
  1. tariff_scraper.py +6 -1
tariff_scraper.py CHANGED
@@ -44,7 +44,12 @@ def scrape_multiple_sections_to_csv(url, output_file="pesco_tariff_data.csv"):
44
 
45
  # Create a dataframe for the current table
46
  df = pd.DataFrame(data[1:], columns=data[0]) if len(data) > 1 else pd.DataFrame(data)
47
- df["Section"] = f"Section {i}" # Add a section label for differentiation
 
 
 
 
 
48
  all_data.append(df)
49
 
50
  # Combine all dataframes into one
 
44
 
45
  # Create a dataframe for the current table
46
  df = pd.DataFrame(data[1:], columns=data[0]) if len(data) > 1 else pd.DataFrame(data)
47
+
48
+ # Ensure column names are unique
49
+ df.columns = pd.io.parsers.ParserBase({'names': df.columns})._maybe_dedup_names()
50
+
51
+ # Add a section identifier
52
+ df["Section"] = f"Section {i}"
53
  all_data.append(df)
54
 
55
  # Combine all dataframes into one