scraper.py

# scraper.py
import requests
from bs4 import BeautifulSoup

def fetch_tariff_data(url):
    try:
        response = requests.get(url)
        response.raise_for_status()
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # Locate the table with tariff data
        table = soup.find('table')
        if not table:
            print("Error: No table found on the page.")
            return None

        tariff_data = []
        rows = table.find_all('tr')[1:]  # Skip the header row

        for row in rows:
            cols = row.find_all('td')
            if len(cols) >= 2:
                particulars = cols[0].text.strip()
                rate = parse_float(cols[-1].text.strip())

                if "Units" in particulars:
                    limits = [parse_float(x) for x in particulars.split(' ')[1:]]
                    lower_limit = limits[0]
                    upper_limit = limits[1] if len(limits) > 1 else float('inf')
                else:
                    lower_limit = 0
                    upper_limit = float('inf')

                tariff_data.append({
                    'lower_limit': lower_limit,
                    'upper_limit': upper_limit,
                    'rate': rate
                })
        
        return tariff_data
    except Exception as e:
        print(f"Error fetching tariff data: {e}")
        return None

def parse_float(value):
    try:
        return float(value.replace(',', '').replace('PKR', '').strip())
    except ValueError:
        return float('inf')