Spaces:
Sleeping
Sleeping
Download fetch.py from gauravmeena0708/epfo-circulars: direct link, hf CLI and curl.
- Browser
- Download file 8.71 kB
-
https://huggingface.co/spaces/gauravmeena0708/epfo-circulars/resolve/main/fetch.py
- Command line
-
hf download hf://spaces/gauravmeena0708/epfo-circulars/fetch.py
-
curl -L -o fetch.py https://huggingface.co/spaces/gauravmeena0708/epfo-circulars/resolve/main/fetch.py
8.71 kB
| """Add newly listed EPFO circulars to circular-data.json. | |
| The issuer lists circulars at https://www.epfo.gov.in/circulars/ (also served at | |
| https://pmvbry.epfindia.gov.in/circulars/). The page carries every listed circular in | |
| its HTML, but only part of the archive, so this script merges: existing records are | |
| never removed or rewritten, and only circulars that are not already held are added. | |
| python fetch.py --dry-run # show what would be added | |
| python fetch.py # update circular-data.json | |
| python fetch.py --html saved.html # read a saved copy of the page | |
| """ | |
| import argparse | |
| from datetime import datetime | |
| import json | |
| import os | |
| import re | |
| import sys | |
| from urllib.parse import unquote, urlparse | |
| from bs4 import BeautifulSoup | |
| import requests | |
| CIRCULAR_DATA_FILE = "circular-data.json" | |
| LISTING_URLS = ( | |
| "https://www.epfo.gov.in/circulars/", | |
| "https://pmvbry.epfindia.gov.in/circulars/", | |
| ) | |
| HEADERS = { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", | |
| "Accept": "text/html,application/xhtml+xml", | |
| "Accept-Language": "en-US,en;q=0.5", | |
| } | |
| # A real listing has well over a thousand entries; far fewer means the page changed. | |
| MIN_EXPECTED_ENTRIES = 500 | |
| DATE = r"\d{1,2}[/.-]\d{1,2}[/.-]\d{4}" | |
| # "... - Regarding No. HRM-I/S-4(1)95/S.L./806 dated 24/10/2025" | |
| # Annexure names can follow the date, so neither pattern is anchored to the end of the title. | |
| NUMBER_AND_DATE = re.compile(rf"\b(?:File\s+)?No\s*[.:]+\s*(?P<number>\S.*?)\s+dated?\s+(?P<date>{DATE})", re.I) | |
| LETTER_DATE = re.compile(rf"\bdated?\s+(?P<date>{DATE})", re.I) | |
| def load_json_file(filepath): | |
| if os.path.exists(filepath): | |
| with open(filepath, "r", encoding="utf-8") as f: | |
| return json.load(f) | |
| return [] | |
| def save_json_file(data, filepath): | |
| with open(filepath, "w", encoding="utf-8") as f: | |
| json.dump(data, f, ensure_ascii=False, indent=4) | |
| print(f"Data saved to {filepath}") | |
| def _date(value): | |
| """Return dd/mm/yyyy for a date written with /, . or -, or '' if it is not a date.""" | |
| match = re.fullmatch(r"(\d{1,2})[/.-](\d{1,2})[/.-](\d{4})", (value or "").strip()) | |
| if not match: | |
| return "" | |
| day, month, year = (int(part) for part in match.groups()) | |
| try: | |
| return datetime(year, month, day).strftime("%d/%m/%Y") | |
| except ValueError: | |
| return "" | |
| def _is_pdf_link(url): | |
| # The page writes "http://false" where no file is attached. | |
| parsed = urlparse(url or "") | |
| return parsed.scheme in ("http", "https") and "." in parsed.netloc and parsed.path.lower().endswith(".pdf") | |
| def parse_listing(page_html): | |
| """Return the circulars on the listing page, in page order (latest first).""" | |
| soup = BeautifulSoup(page_html, "html.parser") | |
| entries = [] | |
| for box in soup.select("div.download-box"): | |
| heading = box.select_one(".download-title") | |
| link = box.select_one("a[href]") | |
| if heading is None: | |
| continue | |
| meta = [item.get_text(" ", strip=True) for item in box.select(".meta-item")] | |
| entries.append({ | |
| "title": " ".join(heading.get_text(" ", strip=True).split()), | |
| "division": meta[0] if meta else "", | |
| "date": _date(meta[1]) if len(meta) > 1 else "", | |
| "url": link["href"].strip() if link else "", | |
| }) | |
| return entries | |
| def to_record(entry): | |
| """Shape a listing entry like the existing circular-data.json records.""" | |
| title = entry["title"] | |
| circular_no, title_date = "", "" | |
| match = NUMBER_AND_DATE.search(title) | |
| if match: | |
| circular_no, title_date = match.group("number").strip(), _date(match.group("date")) | |
| else: | |
| match = LETTER_DATE.search(title) | |
| if match: | |
| title_date = _date(match.group("date")) | |
| return { | |
| "serial_no": "", | |
| "title": title, | |
| "circular_no": circular_no, | |
| # The letter's own date is preferred; the listing date is the upload date when they differ. | |
| "date": title_date or entry["date"], | |
| "hindi_pdf_link": None, | |
| "english_pdf_link": entry["url"], | |
| "division": entry["division"], | |
| } | |
| def _normalise(text): | |
| return re.sub(r"[^0-9a-zऀ-ॿ]+", "", (text or "").casefold()) | |
| def _title_keys(title): | |
| """Keys under which the same circular's title appears in old and new listings. | |
| Older records had the title cut at "No." and the number stored separately; the new | |
| page keeps "... Regarding No. <number> dated <date>" in the title. | |
| """ | |
| # The Hindi copy of a circular is listed separately with "(Hindi)" after the same title. | |
| title = re.sub(r"\(\s*(?:Hindi|English)\s*\)\s*$", "", title.strip(), flags=re.I) | |
| full = _normalise(title) | |
| # Older records cut at "No." and sometimes kept the word before it ("... Regarding File"). | |
| cuts = [_normalise(re.split(pattern, title, maxsplit=1, flags=re.I)[0]) | |
| for pattern in (r"\bNo\s*[.:]", r"\b(?:Circular\s+|File\s+)?No\s*[.:]")] | |
| keys = {full, *cuts} | |
| for key in (full, *cuts): | |
| if len(key) >= 60: | |
| keys.add(key[:60]) | |
| return {key for key in keys if len(key) >= 12} | |
| def _file_key(url): | |
| """PDF file name without the "-1", "-2" suffixes the new site adds to re-uploads.""" | |
| name = unquote(os.path.basename(urlparse(url or "").path)).casefold() | |
| if not name.endswith(".pdf"): | |
| return "" | |
| stem = re.sub(r"(?:-\d{1,2})+$", "", name[:-4]) | |
| # The new site also drops brackets and spaces from file names. | |
| stem = re.sub(r"[^0-9a-zऀ-ॿ]+", "", stem) | |
| # Numbered uploads such as "161.pdf" say nothing about the circular. | |
| return "" if stem.isdigit() or len(stem) < 6 else stem | |
| def merge(existing, listed): | |
| """Return (records, added): existing records unchanged, unseen circulars first.""" | |
| titles, files = set(), set() | |
| for record in existing: | |
| titles |= _title_keys(record.get("title", "")) | |
| for field in ("english_pdf_link", "hindi_pdf_link"): | |
| key = _file_key(record.get(field)) | |
| if key: | |
| files.add(key) | |
| added = [] | |
| for entry in listed: | |
| if not _is_pdf_link(entry["url"]): | |
| continue | |
| title_keys, file_key = _title_keys(entry["title"]), _file_key(entry["url"]) | |
| if title_keys & titles or (file_key and file_key in files): | |
| continue | |
| added.append(to_record(entry)) | |
| titles |= title_keys | |
| if file_key: | |
| files.add(file_key) | |
| return added + list(existing), added | |
| def download_listing(): | |
| last_error = None | |
| for url in LISTING_URLS: | |
| try: | |
| response = requests.get(url, headers=HEADERS, timeout=60) | |
| response.raise_for_status() | |
| response.encoding = "utf-8" | |
| return response.text, url | |
| except requests.exceptions.RequestException as error: | |
| print(f" Could not read {url}: {error}") | |
| last_error = error | |
| raise SystemExit(f"The circular listing could not be downloaded: {last_error}") | |
| def fetch_circular_metadata(html_path=None, data_file=CIRCULAR_DATA_FILE, dry_run=False): | |
| if html_path: | |
| with open(html_path, "r", encoding="utf-8", errors="replace") as f: | |
| page, source = f.read(), html_path | |
| else: | |
| page, source = download_listing() | |
| listed = parse_listing(page) | |
| print(f"{len(listed)} circulars listed at {source}") | |
| if len(listed) < MIN_EXPECTED_ENTRIES: | |
| raise SystemExit("Too few circulars were found; the page layout may have changed. Nothing was written.") | |
| existing = load_json_file(data_file) | |
| records, added = merge(existing, listed) | |
| print(f"{len(existing)} circulars already held; {len(added)} new") | |
| for record in added: | |
| print(f" + {record['date'] or 'undated'} | {record['title'][:110]} | {record['english_pdf_link']}") | |
| if added and not dry_run: | |
| save_json_file(records, data_file) | |
| return added | |
| if __name__ == "__main__": | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| parser = argparse.ArgumentParser(description="Add newly listed EPFO circulars to circular-data.json.") | |
| parser.add_argument("--action", choices=["fetch", "all"], default="fetch", | |
| help="Kept for compatibility; both values fetch the listing.") | |
| parser.add_argument("--html", help="Read a saved copy of the listing page instead of downloading it.") | |
| parser.add_argument("--dry-run", action="store_true", help="Report new circulars without writing the data file.") | |
| args = parser.parse_args() | |
| fetch_circular_metadata(html_path=args.html, dry_run=args.dry_run) | |