split_syllabus.py

# Bulk KTU Syllabus Splitter & Metadata Generator
# Automatically loops through all downloaded branches!

import argparse
import fitz  # PyMuPDF
import json
import os
import re

def clean_subject_name(name):
    name = name.strip()
    name = re.sub(r'^(COURSE NAME\s*:\s*)', '', name, flags=re.IGNORECASE)
    name = re.sub(r'\s+', ' ', name)
    return name

def bulk_split_ktu_syllabus(summary_file, scheme, output_dir):
    pdf_dir = os.path.join(output_dir, "pdfs")
    os.makedirs(pdf_dir, exist_ok=True)
    
    # Read the summary file generated by the scraper
    with open(summary_file, 'r') as f:
        downloaded_files = json.load(f)
        
    flat_db = []
    
    print(f"Found {len(downloaded_files)} branches to process...")
    
    for item in downloaded_files:
        pdf_path = item["local_path"]
        branch = item["branch_name"]
        
        if not os.path.exists(pdf_path):
            print(f"  [Skip] Could not find {pdf_path}")
            continue
            
        print(f"\nProcessing: {branch}")
        doc = fitz.open(pdf_path)
        total_pages = len(doc)
        
        subjects = []
        current_subject = None
        
        for page_idx in range(total_pages):
            page = doc.load_page(page_idx)
            text = page.get_text()
            lines = [line.strip() for line in text.split("\n") if line.strip()]
            
            is_subject_start = False
            sem_tag = None
            course_name = None
            course_code = None
            
            for i, line in enumerate(lines):
                if line.startswith("SEMESTER S") and len(line) <= 12:
                    sem_tag = line
                    if i + 1 < len(lines):
                        course_name = lines[i+1]
                        if i + 2 < len(lines) and (lines[i+2].startswith("(") or "group" in lines[i+2].lower()):
                            course_name += " " + lines[i+2]
                            
                    for j in range(i, min(i + 15, len(lines))):
                        if lines[j] == "Course Code" and j + 1 < len(lines):
                            course_code = lines[j+1]
                            break
                            
                    if course_code:
                        is_subject_start = True
                        break
                        
            if is_subject_start:
                if current_subject:
                    current_subject["end_page"] = page_idx
                    subjects.append(current_subject)
                    
                current_subject = {
                    "semester": sem_tag.replace("SEMESTER ", "").strip(),
                    "subject_name": clean_subject_name(course_name),
                    "course_code": course_code.strip(),
                    "start_page": page_idx + 1,
                    "end_page": None
                }
                
        if current_subject:
            current_subject["end_page"] = total_pages
            subjects.append(current_subject)
            
        print(f"  -> Discovered {len(subjects)} subjects")
        
        for sub in subjects:
            sem = sub["semester"]
            code = sub["course_code"]
            name = sub["subject_name"]
            start = sub["start_page"]
            end = sub["end_page"]
            
            pdf_filename = f"{sem}_{code}.pdf"
            pdf_relative_path = f"pdfs/{pdf_filename}"
            pdf_out_path = os.path.join(pdf_dir, pdf_filename)
            
            # Extract and save the specific pages
            sub_doc = fitz.open()
            sub_doc.insert_pdf(doc, from_page=start-1, to_page=end-1)
            sub_doc.save(pdf_out_path)
            sub_doc.close()
            
            # Append to the massive master array
            flat_db.append({
                "branch": branch,
                "scheme": scheme,
                "semester": sem,
                "course_code": code,
                "subject_name": name,
                "start_page": start,
                "end_page": end,
                "page_count": end - start + 1,
                "pdf_file": pdf_relative_path
            })
        
        doc.close()

    # Outside the loop: Save the combined master database
    with open(os.path.join(output_dir, "syllabus_flat.json"), "w") as jf:
        json.dump(flat_db, jf, indent=2)
        
    print(f"\n=== BULK PROCESS COMPLETED ===")
    print(f"Processed {len(downloaded_files)} branches.")
    print(f"Generated a massive combined database with {len(flat_db)} total subjects!")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="KTU PDF Syllabus Bulk Splitter")
    parser.add_argument("--summary", default="./raw_pdfs/downloaded_syllabi.json", help="Path to the JSON summary generated by scraper")
    parser.add_argument("--scheme", default="B.Tech Full Time 2024 Scheme", help="Syllabus scheme name")
    parser.add_argument("--outdir", default=".", help="Directory to output split PDFs and JSON")
    
    args = parser.parse_args()
    bulk_split_ktu_syllabus(args.summary, args.scheme, args.outdir)