scrape_syllabus.py

#!/usr/bin/env python3
"""
KTU Syllabus PDF Scraper & Downloader
Commit this script to your GitHub repository!

This script uses Playwright to automate the APJ Abdul Kalam Technological University (KTU) website,
locates the B.Tech scheme (default: 2024 Scheme), scans all available branches, detects if a syllabus 
PDF is uploaded, and downloads it.

Requirements:
    pip install playwright
    playwright install chromium

Usage:
    python scrape_syllabus.py --scheme "B.TECH FULL TIME 2024 SCHEME" --outdir "./raw_pdfs"
"""

import asyncio
import os
import argparse
import json
import sys
from playwright.async_api import async_playwright

async def scrape_ktu_syllabi(scheme_name, out_dir, headless, max_branches):
    os.makedirs(out_dir, exist_ok=True)
    
    print(f"Starting KTU Scraper for scheme: '{scheme_name}'")
    print(f"Output directory: '{out_dir}'")
    
    async with async_playwright() as p:
        print("Launching Chromium browser...")
        browser = await p.chromium.launch(headless=headless)
        context = await browser.new_context(
            viewport={"width": 1280, "height": 800},
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
        )
        page = await context.new_page()
        
        # 1. Load the schemes page (FIXED: Changed to domcontentloaded and 90s timeout)
        print("Navigating to KTU Academics Scheme page...")
        try:
            await page.goto("https://ktu.edu.in/academics/scheme", wait_until="domcontentloaded", timeout=90000)
            # Wait an extra 2 seconds to let any dynamic JavaScript render
            await page.wait_for_timeout(2000) 
        except Exception as e:
            print(f"Failed to load KTU website: {e}")
            await browser.close()
            return
        
        # 2. Find and click the 'Branch' link for the targeted scheme
        print(f"Locating block for scheme: '{scheme_name}'...")
        clicked_branch = await page.evaluate("""(targetScheme) => {
            const elements = Array.from(document.querySelectorAll('*'));
            let targetCard = null;
            // Search for the element with the scheme name
            for (let el of elements) {
                if (el.innerText && el.innerText.toUpperCase().includes(targetScheme.toUpperCase())) {
                    if (!targetCard || el.innerText.length < targetCard.innerText.length) {
                        targetCard = el;
                    }
                }
            }
            if (targetCard) {
                let container = targetCard;
                // Go up to find the section/card container
                for (let i = 0; i < 5; i++) {
                    if (container.parentElement && (
                        container.className.includes('card') || 
                        container.className.includes('box') || 
                        container.className.includes('border') || 
                        container.className.includes('p-')
                    )) {
                        container = container.parentElement;
                        break;
                    }
                    if (container.parentElement) container = container.parentElement;
                }
                
                // Find 'Branch' button/link inside this container
                const branchLink = Array.from(container.querySelectorAll('a, button')).find(
                    a => a.innerText.trim() === 'Branch'
                );
                if (branchLink) {
                    branchLink.click();
                    return { success: true, message: "Clicked 'Branch' link." };
                }
                return { success: false, message: `Found card but no 'Branch' link inside container for ${targetScheme}` };
            }
            return { success: false, message: `Scheme card not found for ${targetScheme}` };
        }""", scheme_name)
        
        print("Navigation click result:", clicked_branch)
        if not clicked_branch["success"]:
            print(f"Error: {clicked_branch['message']}", file=sys.stderr)
            await browser.close()
            return
            
        await page.wait_for_timeout(5000)
        print(f"Loaded branch page. Current URL: {page.url}")
        
        # 3. Detect and count all branches
        branches_list = await page.evaluate("""() => {
            const results = [];
            const rows = document.querySelectorAll('.border-bottom-dotted.p-b-10.p-t-5.row');
            rows.forEach((row, idx) => {
                const branchAnchor = row.querySelector('a.font-weight-bold');
                if (branchAnchor) {
                    results.push({
                        index: idx,
                        branchName: branchAnchor.innerText.trim()
                    });
                }
            });
            return results;
        }""")
        
        total_branches = len(branches_list)
        print(f"Discovered {total_branches} branches under this scheme.")
        
        if max_branches:
            branches_list = branches_list[:max_branches]
            print(f"Limiting scrape to first {len(branches_list)} branches (test run).")
            
        downloaded_summary = []
        
        # Helper to load branch page fresh if needed
        async def ensure_on_branch_page():
            if "/academics/branch" not in page.url:
                print("Not on branch page. Reloading branch list...")
                # FIX: Changed to domcontentloaded and 90s timeout
                await page.goto("https://ktu.edu.in/academics/scheme", wait_until="domcontentloaded", timeout=90000)
                await page.wait_for_timeout(2000)
                
                await page.evaluate("""(targetScheme) => {
                    const elements = Array.from(document.querySelectorAll('*'));
                    let targetCard = null;
                    for (let el of elements) {
                        if (el.innerText && el.innerText.toUpperCase().includes(targetScheme.toUpperCase())) {
                            if (!targetCard || el.innerText.length < targetCard.innerText.length) {
                                targetCard = el;
                            }
                        }
                    }
                    if (targetCard) {
                        let container = targetCard;
                        for (let i = 0; i < 5; i++) {
                            if (container.parentElement && (container.className.includes('card') || container.className.includes('box') || container.className.includes('border') || container.className.includes('p-'))) {
                                container = container.parentElement;
                                break;
                            }
                            if (container.parentElement) container = container.parentElement;
                        }
                        const branchLink = Array.from(container.querySelectorAll('a, button')).find(a => a.innerText.trim() === 'Branch');
                        if (branchLink) branchLink.click();
                    }
                }""", scheme_name)
                await page.wait_for_timeout(5000)
        
        # 4. Loop through branches and download available PDFs
        for b in branches_list:
            idx = b["index"]
            b_name = b["branchName"]
            
            await ensure_on_branch_page()
            
            print(f"\n--- [{idx+1}/{total_branches}] Checking branch: {b_name} ---")
            
            # Click on 'Syllabus' link for this specific branch
            clicked_syl = await page.evaluate("""(rowIdx) => {
                const rows = document.querySelectorAll('.border-bottom-dotted.p-b-10.p-t-5.row');
                if (rows.length > rowIdx) {
                    const btn = Array.from(rows[rowIdx].querySelectorAll('a, button, [role="button"]')).find(
                        el => el.innerText.trim() === 'Syllabus'
                    );
                    if (btn) {
                        btn.click();
                        return true;
                    }
                }
                return false;
            }""", idx)
            
            if not clicked_syl:
                print(f"  [Skip] No syllabus button found in row {idx+1}")
                continue
                
            await page.wait_for_timeout(3500)
            
            # Check if syllabus download button is present
            has_syllabus_btn = await page.evaluate("""() => {
                const btns = Array.from(document.querySelectorAll('button')).filter(btn => {
                    const text = btn.innerText ? btn.innerText.trim() : '';
                    return text === 'Syllabus' && btn.className.includes('btn-hover-blue');
                });
                return btns.length > 0;
            }""")
            
            if not has_syllabus_btn:
                print(f"  [Skip] No syllabus PDF file uploaded on page for this branch.")
                await page.go_back()
                await page.wait_for_timeout(1500)
                continue
                
            # Perform download
            print("  Syllabus PDF detected! Starting download...")
            try:
                # FIX: Increased download timeout to 45 seconds for large PDFs
                async with page.expect_download(timeout=45000) as download_info:
                    await page.evaluate("""() => {
                        const btns = Array.from(document.querySelectorAll('button')).filter(btn => {
                            const text = btn.innerText ? btn.innerText.trim() : '';
                            return text === 'Syllabus' && btn.className.includes('btn-hover-blue');
                        });
                        if (btns.length > 0) {
                            btns[0].click();
                        }
                    }""")
                    
                download = await download_info.value
                suggested_fn = download.suggested_filename
                
                # Make filename safe and save
                out_path = os.path.join(out_dir, suggested_fn)
                await download.save_as(out_path)
                
                size = os.path.getsize(out_path)
                print(f"  [SUCCESS] Saved PDF to {out_path} ({size} bytes)")
                
                downloaded_summary.append({
                    "branch_name": b_name,
                    "filename": suggested_fn,
                    "local_path": out_path,
                    "file_size": size
                })
                
            except Exception as e:
                print(f"  [ERROR] Failed downloading syllabus for {b_name}: {e}")
                
            # Go back to branch list
            print("  Returning to branch page...")
            await page.go_back()
            await page.wait_for_timeout(2000)
            
        # 5. Output summary
        summary_path = os.path.join(out_dir, "downloaded_syllabi.json")
        with open(summary_path, "w") as jf:
            json.dump(downloaded_summary, jf, indent=2)
            
        print("\n" + "="*40)
        print("Syllabus scraping completed!")
        print(f"Successfully downloaded {len(downloaded_summary)} files.")
        print(f"Summary written to: {summary_path}")
        print("="*40)
        
        await browser.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="KTU Website Playwright Syllabus Scraper")
    parser.add_argument("--scheme", default="B.TECH FULL TIME 2024 SCHEME", help="Target scheme name to scrape (e.g. B.TECH FULL TIME 2024 SCHEME)")
    parser.add_argument("--outdir", default="./raw_pdfs", help="Directory where raw syllabus PDFs should be downloaded")
    parser.add_argument("--headless", type=bool, default=True, help="Run browser in headless mode")
    parser.add_argument("--max-branches", type=int, default=None, help="Limit number of branches to scrape for testing")
    
    args = parser.parse_args()
    
    # Run the async loop
    asyncio.run(scrape_ktu_syllabi(
        scheme_name=args.scheme,
        out_dir=args.outdir,
        headless=args.headless,
        max_branches=args.max_branches
    ))