scrape_playwright.py

#!/usr/bin/env python3
"""
scrape_playwright.py
Uses Playwright (headless Firefox) to fully render education.gov.za pages
and extract all dynamically-loaded document links.

Usage:
    cd ~/leotsha_project
    python3 scrape_playwright.py

Playwright renders the full page (JS executed), then extracts ALL links.
"""

import os, re, csv, time, requests
from urllib.parse import urljoin, urlparse
from playwright.sync_api import sync_playwright

BASE_DIR = os.path.expanduser("~/leotsha_project/corpus")

SEED_URLS = [
    "https://www.education.gov.za/SelfStudyGuidesGrade10-12.aspx",
    "https://www.education.gov.za/Resources/Manuals.aspx",
    "https://www.education.gov.za/Resources/Publications.aspx",
    "https://www.education.gov.za/Resources/Legislation/WhitePapers.aspx",
    "https://www.education.gov.za/Resources/Policies.aspx",
    "https://www.education.gov.za/Curriculum/NationalCirculars.aspx",
    "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/LTSMNationalCatalogue.aspx",
    "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/DigitalContent.aspx",
    "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/Workbooks.aspx",
    "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/MindtheGapStudyGuides.aspx",
    "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/GradedReadersandBigBookHL.aspx",
    "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/IIALResources.aspx",
    "https://www.education.gov.za/Curriculum/LearningandTeachingSupportMaterials(LTSM)/AttheCrossroadsTextbooks.aspx",
    "https://www.education.gov.za/Curriculum/CurriculumAssessmentPolicyStatements(CAPS).aspx",
    "https://www.education.gov.za/Informationfor/Researcher.aspx",
    "https://www.justice.gov.za/legislation/constitution/index.html",
    "https://www.gov.za/about-government/government-system/local-government",
]

DOC_EXTENSIONS = [".pdf", ".doc", ".docx", ".pptx", ".xlsx", ".zip"]

SEPEDI_KEYWORDS = [
    "sepedi", "sesotho sa leboa", "northern sotho", "nsotho",
    "n.sotho", "sesotho", "leboa"
]

GRADE_KEYWORDS = [
    "grade 10", "grade 11", "grade 12", "gr 10", "gr 11", "gr 12",
    "fet", "grade10", "grade11", "grade12", "home language", "hl",
    "fal", "first additional", "caps", "mind the gap", "self study",
    "workbook", "ltsm", "curriculum"
]

def is_doc(url):
    return any(url.lower().endswith(ext) for ext in DOC_EXTENSIONS)

def is_relevant(url, text):
    combined = (url + " " + text).lower()
    return (any(k in combined for k in SEPEDI_KEYWORDS) or
            any(k in combined for k in GRADE_KEYWORDS))

def is_sepedi(url, text):
    combined = (url + " " + text).lower()
    return any(k in combined for k in SEPEDI_KEYWORDS)

def get_folder(url, text):
    c = (url + " " + text).lower()
    if "mindthegap" in c or "mind the gap" in c: return "eduintel/mind_the_gap"
    if "workbook" in c:                           return "eduintel/workbooks"
    if "selfstud" in c or "self study" in c:      return "eduintel/self_study"
    if "caps" in c or "curriculum" in c:          return "eduintel/caps"
    if "catalogue" in c or "catalog" in c:        return "eduintel/catalogues"
    if "constitution" in c or "billofright" in c: return "wardintel"
    if "education.gov.za" in url:                 return "eduintel/other"
    return "other"

def safe_filename(url):
    name = os.path.basename(urlparse(url).path)
    name = re.sub(r'[^\w\-_\. ]', '_', name)
    return name if name else "document.pdf"

def download_file(url, dest):
    try:
        r = requests.get(url, headers={"User-Agent": "Mozilla/5.0"},
                        timeout=30, stream=True)
        r.raise_for_status()
        os.makedirs(os.path.dirname(dest), exist_ok=True)
        with open(dest, "wb") as f:
            for chunk in r.iter_content(8192):
                f.write(chunk)
        return os.path.getsize(dest) // 1024
    except Exception as e:
        print(f"      ✗ Download error: {e}")
        return None

def scrape_page_playwright(page, url):
    """Render a page with Playwright and extract all document links."""
    docs = []
    try:
        page.goto(url, wait_until="networkidle", timeout=30000)

        # Scroll to bottom to trigger lazy-loaded content
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        time.sleep(2)

        # Try clicking any "expand all" or accordion buttons
        for selector in [
            "a[class*='expand']", "button[class*='expand']",
            "a[class*='accordion']", ".accordion-toggle",
            "[data-toggle='collapse']"
        ]:
            try:
                elements = page.query_selector_all(selector)
                for el in elements[:10]:
                    el.click()
                time.sleep(1)
            except:
                pass

        # Extract all links from fully rendered DOM
        links = page.evaluate("""
            () => Array.from(document.querySelectorAll('a[href]'))
                .map(a => ({href: a.href, text: a.innerText.trim()}))
        """)

        for link in links:
            href = link.get("href", "")
            text = link.get("text", "")
            if href and is_doc(href):
                docs.append({"url": href, "text": text, "source": url})

    except Exception as e:
        print(f"    ✗ Playwright error: {e}")

    return docs

def main():
    print("\n" + "━"*62)
    print("  SEDIBA SYSTEMS — Playwright Corpus Scraper")
    print(f"  Output: {BASE_DIR}")
    print("  Using: Headless Firefox (JavaScript rendering enabled)")
    print("━"*62 + "\n")

    # Create folders
    for f in ["eduintel/caps","eduintel/workbooks","eduintel/mind_the_gap",
              "eduintel/self_study","eduintel/catalogues","eduintel/other",
              "wardintel","other"]:
        os.makedirs(os.path.join(BASE_DIR, f), exist_ok=True)

    all_docs  = []
    seen_urls = set()
    log_rows  = []

    # ── Phase 1: Crawl with Playwright ──────────────────────
    print("PHASE 1 — Rendering pages with Playwright Firefox...\n")

    with sync_playwright() as p:
        browser = p.firefox.launch(headless=True)
        context = browser.new_context(
            user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
        )
        page = context.new_page()

        for seed in SEED_URLS:
            print(f"  Rendering: {seed}")
            docs = scrape_page_playwright(page, seed)
            new_docs = 0
            for d in docs:
                if d["url"] not in seen_urls:
                    seen_urls.add(d["url"])
                    d["relevant"] = is_relevant(d["url"], d["text"])
                    d["priority"] = is_sepedi(d["url"], d["text"])
                    d["folder"]   = get_folder(d["url"], d["text"])
                    all_docs.append(d)
                    new_docs += 1

            sepedi_count = sum(1 for d in docs if is_sepedi(d["url"], d["text"]))
            print(f"    → Found {new_docs} document links "
                  f"({sepedi_count} Sepedi priority)")
            time.sleep(3)

        page.close()
        browser.close()

    relevant = [d for d in all_docs if d["relevant"]]
    priority = [d for d in all_docs if d["priority"]]

    print(f"\n  Total docs found:     {len(all_docs)}")
    print(f"  Grade 10-12 relevant: {len(relevant)}")
    print(f"  ★ Sepedi priority:    {len(priority)}")

    # ── Phase 2: Download relevant docs ─────────────────────
    print("\nPHASE 2 — Downloading documents...\n")

    downloaded = 0
    skipped    = 0
    failed     = 0

    # Priority first: Sepedi docs, then general Grade 10-12
    to_download = sorted(relevant, key=lambda d: (not d["priority"], d["text"].lower()))

    for doc in to_download:
        fname = safe_filename(doc["url"])
        dest  = os.path.join(BASE_DIR, doc["folder"], fname)

        tag = "★ SEPEDI" if doc["priority"] else "  Gr10-12"
        print(f"  [{tag}] {doc['text'][:55]}")

        if os.path.exists(dest):
            print(f"           ⏭ Already exists — skipping")
            skipped += 1
            log_rows.append({**doc,"status":"exists","size_kb":os.path.getsize(dest)//1024,"local_path":dest})
            continue

        size_kb = download_file(doc["url"], dest)
        if size_kb is not None:
            print(f"           ✓ {doc['folder']}/ ({size_kb} KB)")
            log_rows.append({**doc,"status":"downloaded","size_kb":size_kb,"local_path":dest})
            downloaded += 1
        else:
            log_rows.append({**doc,"status":"failed","size_kb":0,"local_path":""})
            failed += 1
        time.sleep(1)

    # Also log all found-but-not-downloaded docs
    for doc in all_docs:
        if not doc["relevant"]:
            log_rows.append({**doc,"status":"skipped_irrelevant","size_kb":0,"local_path":""})

    # ── Write CSV index ──────────────────────────────────────
    log_path = os.path.join(BASE_DIR, "_index.csv")
    with open(log_path, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=[
            "status","priority","relevant","folder",
            "text","url","source","size_kb","local_path"
        ])
        writer.writeheader()
        writer.writerows(log_rows)

    # ── Summary ──────────────────────────────────────────────
    print("\n" + "━"*62)
    print("  COMPLETE")
    print(f"  All documents found:     {len(all_docs)}")
    print(f"  Sepedi priority docs:    {len(priority)}")
    print(f"  Downloaded:              {downloaded}")
    print(f"  Already existed:         {skipped}")
    print(f"  Failed:                  {failed}")
    print(f"  Index:                   {log_path}")
    print("")
    print("  Review _index.csv for the full list of everything found.")
    print("  Sepedi files are in: corpus/eduintel/ and corpus/wardintel/")
    print("━"*62 + "\n")

if __name__ == "__main__":
    main()