#!/usr/bin/env python3 """ scrape_playwright.py Uses Playwright (headless Firefox) to fully render education.gov.za pages and extract all dynamically-loaded document links. Usage: cd ~/leotsha_project python3 scrape_playwright.py Playwright renders the full page (JS executed), then extracts ALL links. """ import os, re, csv, time, requests from urllib.parse import urljoin, urlparse from playwright.sync_api import sync_playwright BASE_DIR = os.path.expanduser("~/leotsha_project/corpus") SEED_URLS = [ "https://www.education.gov.za/SelfStudyGuidesGrade10-12.aspx", "https://www.education.gov.za/Resources/Manuals.aspx", "https://www.education.gov.za/Resources/Publications.aspx", "https://www.education.gov.za/Resources/Legislation/WhitePapers.aspx", "https://www.education.gov.za/Resources/Policies.aspx", "https://www.education.gov.za/Curriculum/NationalCirculars.aspx", "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/LTSMNationalCatalogue.aspx", "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/DigitalContent.aspx", "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/Workbooks.aspx", "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/MindtheGapStudyGuides.aspx", "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/GradedReadersandBigBookHL.aspx", "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/IIALResources.aspx", "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/AttheCrossroadsTextbooks.aspx", "https://www.education.gov.za/Curriculum/CurriculumAssessmentPolicyStatements(CAPS).aspx", "https://www.education.gov.za/Informationfor/Researcher.aspx", "https://www.justice.gov.za/legislation/constitution/index.html", "https://www.gov.za/about-government/government-system/local-government", ] DOC_EXTENSIONS = [".pdf", ".doc", ".docx", ".pptx", ".xlsx", ".zip"] SEPEDI_KEYWORDS = [ "sepedi", "sesotho sa leboa", "northern sotho", "nsotho", "n.sotho", "sesotho", "leboa" ] GRADE_KEYWORDS = [ "grade 10", "grade 11", "grade 12", "gr 10", "gr 11", "gr 12", "fet", "grade10", "grade11", "grade12", "home language", "hl", "fal", "first additional", "caps", "mind the gap", "self study", "workbook", "ltsm", "curriculum" ] def is_doc(url): return any(url.lower().endswith(ext) for ext in DOC_EXTENSIONS) def is_relevant(url, text): combined = (url + " " + text).lower() return (any(k in combined for k in SEPEDI_KEYWORDS) or any(k in combined for k in GRADE_KEYWORDS)) def is_sepedi(url, text): combined = (url + " " + text).lower() return any(k in combined for k in SEPEDI_KEYWORDS) def get_folder(url, text): c = (url + " " + text).lower() if "mindthegap" in c or "mind the gap" in c: return "eduintel/mind_the_gap" if "workbook" in c: return "eduintel/workbooks" if "selfstud" in c or "self study" in c: return "eduintel/self_study" if "caps" in c or "curriculum" in c: return "eduintel/caps" if "catalogue" in c or "catalog" in c: return "eduintel/catalogues" if "constitution" in c or "billofright" in c: return "wardintel" if "education.gov.za" in url: return "eduintel/other" return "other" def safe_filename(url): name = os.path.basename(urlparse(url).path) name = re.sub(r'[^\w\-_\. ]', '_', name) return name if name else "document.pdf" def download_file(url, dest): try: r = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=30, stream=True) r.raise_for_status() os.makedirs(os.path.dirname(dest), exist_ok=True) with open(dest, "wb") as f: for chunk in r.iter_content(8192): f.write(chunk) return os.path.getsize(dest) // 1024 except Exception as e: print(f" ✗ Download error: {e}") return None def scrape_page_playwright(page, url): """Render a page with Playwright and extract all document links.""" docs = [] try: page.goto(url, wait_until="networkidle", timeout=30000) # Scroll to bottom to trigger lazy-loaded content page.evaluate("window.scrollTo(0, document.body.scrollHeight)") time.sleep(2) # Try clicking any "expand all" or accordion buttons for selector in [ "a[class*='expand']", "button[class*='expand']", "a[class*='accordion']", ".accordion-toggle", "[data-toggle='collapse']" ]: try: elements = page.query_selector_all(selector) for el in elements[:10]: el.click() time.sleep(1) except: pass # Extract all links from fully rendered DOM links = page.evaluate(""" () => Array.from(document.querySelectorAll('a[href]')) .map(a => ({href: a.href, text: a.innerText.trim()})) """) for link in links: href = link.get("href", "") text = link.get("text", "") if href and is_doc(href): docs.append({"url": href, "text": text, "source": url}) except Exception as e: print(f" ✗ Playwright error: {e}") return docs def main(): print("\n" + "━"*62) print(" SEDIBA SYSTEMS — Playwright Corpus Scraper") print(f" Output: {BASE_DIR}") print(" Using: Headless Firefox (JavaScript rendering enabled)") print("━"*62 + "\n") # Create folders for f in ["eduintel/caps","eduintel/workbooks","eduintel/mind_the_gap", "eduintel/self_study","eduintel/catalogues","eduintel/other", "wardintel","other"]: os.makedirs(os.path.join(BASE_DIR, f), exist_ok=True) all_docs = [] seen_urls = set() log_rows = [] # ── Phase 1: Crawl with Playwright ────────────────────── print("PHASE 1 — Rendering pages with Playwright Firefox...\n") with sync_playwright() as p: browser = p.firefox.launch(headless=True) context = browser.new_context( user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0" ) page = context.new_page() for seed in SEED_URLS: print(f" Rendering: {seed}") docs = scrape_page_playwright(page, seed) new_docs = 0 for d in docs: if d["url"] not in seen_urls: seen_urls.add(d["url"]) d["relevant"] = is_relevant(d["url"], d["text"]) d["priority"] = is_sepedi(d["url"], d["text"]) d["folder"] = get_folder(d["url"], d["text"]) all_docs.append(d) new_docs += 1 sepedi_count = sum(1 for d in docs if is_sepedi(d["url"], d["text"])) print(f" → Found {new_docs} document links " f"({sepedi_count} Sepedi priority)") time.sleep(3) page.close() browser.close() relevant = [d for d in all_docs if d["relevant"]] priority = [d for d in all_docs if d["priority"]] print(f"\n Total docs found: {len(all_docs)}") print(f" Grade 10-12 relevant: {len(relevant)}") print(f" ★ Sepedi priority: {len(priority)}") # ── Phase 2: Download relevant docs ───────────────────── print("\nPHASE 2 — Downloading documents...\n") downloaded = 0 skipped = 0 failed = 0 # Priority first: Sepedi docs, then general Grade 10-12 to_download = sorted(relevant, key=lambda d: (not d["priority"], d["text"].lower())) for doc in to_download: fname = safe_filename(doc["url"]) dest = os.path.join(BASE_DIR, doc["folder"], fname) tag = "★ SEPEDI" if doc["priority"] else " Gr10-12" print(f" [{tag}] {doc['text'][:55]}") if os.path.exists(dest): print(f" ⏭ Already exists — skipping") skipped += 1 log_rows.append({**doc,"status":"exists","size_kb":os.path.getsize(dest)//1024,"local_path":dest}) continue size_kb = download_file(doc["url"], dest) if size_kb is not None: print(f" ✓ {doc['folder']}/ ({size_kb} KB)") log_rows.append({**doc,"status":"downloaded","size_kb":size_kb,"local_path":dest}) downloaded += 1 else: log_rows.append({**doc,"status":"failed","size_kb":0,"local_path":""}) failed += 1 time.sleep(1) # Also log all found-but-not-downloaded docs for doc in all_docs: if not doc["relevant"]: log_rows.append({**doc,"status":"skipped_irrelevant","size_kb":0,"local_path":""}) # ── Write CSV index ────────────────────────────────────── log_path = os.path.join(BASE_DIR, "_index.csv") with open(log_path, "w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=[ "status","priority","relevant","folder", "text","url","source","size_kb","local_path" ]) writer.writeheader() writer.writerows(log_rows) # ── Summary ────────────────────────────────────────────── print("\n" + "━"*62) print(" COMPLETE") print(f" All documents found: {len(all_docs)}") print(f" Sepedi priority docs: {len(priority)}") print(f" Downloaded: {downloaded}") print(f" Already existed: {skipped}") print(f" Failed: {failed}") print(f" Index: {log_path}") print("") print(" Review _index.csv for the full list of everything found.") print(" Sepedi files are in: corpus/eduintel/ and corpus/wardintel/") print("━"*62 + "\n") if __name__ == "__main__": main()