| |
| import requests, re |
| from bs4 import BeautifulSoup |
| import pandas as pd |
| from urllib.parse import urljoin |
|
|
| HEADERS = { |
| "User-Agent": "Mozilla/5.0 (compatible; TenderScraper/1.0; +https://huggingface.co/spaces)" |
| } |
|
|
| def parse_value(text: str) -> str: |
| """Normalise value text like '15 Lakhs' → '15 Lakhs' (strip ₹ icon).""" |
| return re.sub(r"[^\d.,A-Za-z ]+", "", text).strip() |
|
|
| def scrape_tender_list(url: str) -> pd.DataFrame: |
| """Return a DataFrame with one row per <div class='tender_row'>.""" |
| resp = requests.get(url, headers=HEADERS, timeout=20) |
| resp.raise_for_status() |
| soup = BeautifulSoup(resp.text, "html.parser") |
|
|
| rows = [] |
| for tr in soup.select("div.tender_row"): |
| |
| header = tr.select_one("h2.workDesc strong") |
| if not header: |
| continue |
| parts = [t.strip() for t in header.stripped_strings if t.strip()] |
| authority = parts[0] if parts else "" |
| city = parts[1][2:] if len(parts) > 1 else "" |
| state = parts[2][2:] if len(parts) > 2 else "" |
|
|
| |
| a = tr.select_one("a.m-brief") |
| tender_id = a.select_one("span.m-tender-id").text.strip() if a else "" |
| work = a.get_text(" ", strip=True).replace(tender_id, "").strip() if a else "" |
| tender_url = urljoin(url, a["href"]) if a else "" |
|
|
| |
| date_span = tr.select_one("span.m-due-date") |
| if date_span: |
| due_text = " ".join( |
| s.get_text(strip=True) |
| for s in date_span.parent.select("span")[1:] |
| ) |
| else: |
| due_text = "" |
|
|
| |
| value_span = tr.select_one("span.m-value") |
| value_text = parse_value(value_span.parent.get_text(" ", strip=True)) if value_span else "" |
|
|
| rows.append({ |
| "Authority": authority, |
| "City": city, |
| "State": state, |
| "Tender ID": tender_id, |
| "Work Description": work, |
| "Due Date": due_text, |
| "Estimated Value": value_text, |
| "Notice URL": tender_url, |
| }) |
|
|
| return pd.DataFrame(rows) |
|
|
