ingestion/parser.py

"""
Parser: Converts raw HTML → clean text + structured metadata
Output: JSON files in /data/parsed/
"""

import json
import re
from pathlib import Path
from bs4 import BeautifulSoup

RAW_DIR = Path("data/raw")
PARSED_DIR = Path("data/parsed")
PARSED_DIR.mkdir(parents=True, exist_ok=True)

def clean_text(text: str) -> str:
    text = re.sub(r'\s+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()

def parse_legislation(html: str, meta: dict) -> dict:
    soup = BeautifulSoup(html, "lxml")

    for tag in soup(["nav", "footer", "script", "style", "header"]):
        tag.decompose()

    content_div = (
        soup.select_one(".akn-act") or
        soup.select_one(".document-content") or
        soup.select_one("main") or
        soup.body
    )

    raw_text = content_div.get_text(separator="\n") if content_div else ""
    cleaned = clean_text(raw_text)
    sections = extract_sections(cleaned)

    return {
        "title": meta.get("title", ""),
        "url": meta.get("url", ""),
        "type": "legislation",
        "full_text": cleaned,
        "sections": sections,
        "char_count": len(cleaned)
    }

def parse_case_law(html: str, meta: dict) -> dict:
    soup = BeautifulSoup(html, "lxml")

    for tag in soup(["nav", "footer", "script", "style", "header"]):
        tag.decompose()

    content_div = (
        soup.select_one(".akn-judgment") or
        soup.select_one(".judgment-body") or
        soup.select_one(".akn-act") or
        soup.select_one("main") or
        soup.body
    )

    raw_text = content_div.get_text(separator="\n") if content_div else ""
    cleaned = clean_text(raw_text)

    citation = extract_citation(soup)
    court = extract_court(soup)
    date = extract_date(soup)

    return {
        "title": meta.get("title", ""),
        "url": meta.get("url", ""),
        "type": "case_law",
        "citation": citation,
        "court": court,
        "date": date,
        "full_text": cleaned,
        "char_count": len(cleaned)
    }

def extract_sections(text: str) -> list[dict]:
    sections = []
    pattern = re.compile(
        r'(?:^|\n)((?:Section\s+)?\d+[A-Z]?\.)\s+(.+?)(?=\n(?:(?:Section\s+)?\d+[A-Z]?\.|$))',
        re.DOTALL | re.MULTILINE
    )
    for match in pattern.finditer(text):
        section_num = match.group(1).strip()
        section_text = match.group(2).strip()
        if len(section_text) > 10:
            sections.append({
                "section": section_num,
                "text": section_text[:2000]
            })
    return sections

def extract_citation(soup: BeautifulSoup) -> str:
    for selector in [".citation", ".case-citation", "h2", "h3"]:
        el = soup.select_one(selector)
        if el:
            text = el.get_text(strip=True)
            if re.search(r'\[\d{4}\]', text):
                return text
    return ""

def extract_court(soup: BeautifulSoup) -> str:
    for selector in [".court-name", ".court", "[class*='court']"]:
        el = soup.select_one(selector)
        if el:
            return el.get_text(strip=True)
    return ""

def extract_date(soup: BeautifulSoup) -> str:
    for selector in [".date", ".judgment-date", "[class*='date']"]:
        el = soup.select_one(selector)
        if el:
            return el.get_text(strip=True)
    return ""

def parse_all():
    html_files = list(RAW_DIR.glob("*.html"))
    print(f"Parsing {len(html_files)} files...")

    for html_path in html_files:
        meta_path = html_path.with_suffix(".json")

        if not meta_path.exists():
            print(f"  No metadata for {html_path.name}, skipping.")
            continue

        meta = json.loads(meta_path.read_text())
        html = html_path.read_text(encoding="utf-8")

        try:
            if meta["type"] == "legislation":
                parsed = parse_legislation(html, meta)
            elif meta["type"] == "case_law":
                parsed = parse_case_law(html, meta)
            else:
                continue

            out_path = PARSED_DIR / html_path.with_suffix(".json").name
            out_path.write_text(json.dumps(parsed, indent=2, ensure_ascii=False))
            print(f"  Parsed: {parsed['title'][:50]} ({parsed['char_count']} chars)")

        except Exception as e:
            print(f"  ERROR parsing {html_path.name}: {e}")

    print(f"\nParsed files saved to {PARSED_DIR}/")

if __name__ == "__main__":
    parse_all()