src/scraper/html_scraper.py

"""

html_scraper.py

───────────────

Downloads a law page from legislatie.just.ro and splits it into articles.



URL pattern:  https://legislatie.just.ro/Public/DetaliiDocument/{id}



What this file does, step by step:

  1. fetch_law_page()   — downloads the HTML, handles errors & rate limits

  2. extract_title()    — pulls the law title out of the page

  3. extract_raw_text() — strips all HTML tags, keeps only readable text

  4. split_into_articles() — finds "Articolul X" headers and cuts the text there



Install: pip install requests beautifulsoup4

"""

import re
import time
import random
import requests
from bs4 import BeautifulSoup

# ── Constants ─────────────────────────────────────────────────────────────────

# The portal has TWO different URL patterns — we try both automatically.
# DetaliiDocument works for most (A) and (R) versions.
# DetaliiDocumentAfis works for older/original versions and some republications.
URL_PATTERNS = [
    "https://legislatie.just.ro/Public/DetaliiDocument/{}",
    "https://legislatie.just.ro/Public/DetaliiDocumentAfis/{}",
]
BASE_URL = URL_PATTERNS[0]  # kept for backward compatibility with scrape_law()

# We use ONE persistent session for the entire scraping run.
# This is important because:
#   - It reuses the TCP connection (faster, lower overhead on their server)
#   - It automatically handles cookies (looks more like a real browser)
#   - Servers are far less likely to block a session that behaves like a browser
SESSION = requests.Session()
SESSION.headers.update({
    # Identify ourselves as a modern Chrome browser on Windows
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/124.0.0.0 Safari/537.36"
    ),
    # Tell the server we accept HTML and compressed responses
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "ro-RO,ro;q=0.9,en-US;q=0.8,en;q=0.7",
    "Accept-Encoding": "gzip, deflate, br",
    # This is the key anti-block header:
    # It tells the server we "came from" the portal homepage,
    # which is what a real user clicking a search result would look like.
    "Referer": "https://legislatie.just.ro/",
    "Connection": "keep-alive",
})


# ── Step 1: Download the page ─────────────────────────────────────────────────

def _fetch_single_url(url: str, retries: int = 3) -> requests.Response | None:
    """

    Try to GET one URL with retries and rate-limit handling.

    Returns the Response object on success, None on permanent failure.

    """
    for attempt in range(retries):
        try:
            resp = SESSION.get(url, timeout=20)

            if resp.status_code == 429:
                wait = 60 * (attempt + 1)
                print(f"\n  ⚠ Rate limited (429). Waiting {wait}s...")
                time.sleep(wait)
                continue

            if resp.status_code == 403:
                return None  # access denied — caller will try alternate URL

            resp.raise_for_status()
            resp.encoding = resp.apparent_encoding
            return resp

        except requests.exceptions.Timeout:
            time.sleep(2 ** attempt)
        except requests.exceptions.ConnectionError:
            time.sleep(2 ** attempt)
        except requests.exceptions.HTTPError:
            time.sleep(2 ** attempt)

    return None


def fetch_law_page(law_id: int) -> tuple[BeautifulSoup, str] | tuple[None, None]:
    """

    Download the HTML page for one law, trying both URL patterns automatically.



    The portal uses two different URL patterns:

      - DetaliiDocument/{id}     — works for most actualizat (A) versions

      - DetaliiDocumentAfis/{id} — works for republicat (R) and older versions



    We try DetaliiDocument first. If it returns a page with < 500 chars of

    actual text content, we assume it's a redirect/error and try the Afis URL.



    Returns (BeautifulSoup, url_used) on success, or (None, None) on failure.

    """
    for url_pattern in URL_PATTERNS:
        url = url_pattern.format(law_id)
        resp = _fetch_single_url(url)

        if resp is None:
            continue  # try next pattern

        soup = BeautifulSoup(resp.text, "html.parser")

        # Quick sanity check: extract text and see if there's real content.
        # An error/redirect page has very little text (< 500 chars).
        # A real law page has thousands of chars.
        body = soup.find("body")
        text_preview = body.get_text() if body else ""
        if len(text_preview.strip()) >= 500:
            return soup, url  # success — found a real page

        # This URL returned an empty/redirect page — try the next pattern
        print(f"\n  ↻ {url_pattern.split('/')[5]} returned empty page, "
              f"trying alternate URL pattern...")

    print(f"\n  ✗ Both URL patterns failed for ID {law_id}. Skipping.")
    return None, None


# ── Step 2: Extract the title ─────────────────────────────────────────────────

def extract_title(soup: BeautifulSoup) -> str:
    """

    Pull the law's title from the HTML.



    The page <title> tag contains things like:

        "LEGE 53 28/06/2003 - Portal Legislativ"

        "CODUL CIVIL din 17 iulie 2009 - Portal Legislativ"



    We strip the " - Portal Legislativ" suffix and return the rest.

    If the title tag is missing for some reason, we fall back to "Unknown".

    """
    # Primary: use the <title> tag
    title_tag = soup.find("title")
    if title_tag:
        title = title_tag.get_text(strip=True)
        title = title.replace(" - Portal Legislativ", "").strip()
        if title:
            return title

    # Fallback: look for the biggest heading on the page
    for tag in ["h1", "h2", "h3"]:
        heading = soup.find(tag)
        if heading:
            text = heading.get_text(strip=True)
            if len(text) > 5:  # ignore empty or trivial headings
                return text

    return "Unknown"


# ── Step 3: Extract readable text ─────────────────────────────────────────────

def extract_raw_text(soup: BeautifulSoup) -> str:
    """

    Strip all HTML and return the plain text content of the law.



    We remove:

      - <script> and <style> tags (JavaScript and CSS — not law text)

      - <nav> tags (site navigation menus)

      - <footer> and <header> tags (site chrome)

      - <form> tags (search boxes, login forms)

      - Tags with class names that suggest UI elements, not content



    Then we use BeautifulSoup's get_text() to extract the remaining text,

    using newlines as separators so article structure is preserved.

    """
    # Remove all non-content tags completely
    for tag in soup(["script", "style", "nav", "footer", "header", "form"]):
        tag.decompose()

    # Also remove common UI element classes the portal uses
    for tag in soup.find_all(class_=re.compile(
        r"(menu|navbar|breadcrumb|pagination|sidebar|cookie|banner|btn|button)",
        re.IGNORECASE
    )):
        tag.decompose()

    # Find the main content area
    # The portal wraps the law text in a <div> — we try to find it
    # If we can't, we fall back to the full <body>
    content = (
        soup.find("div", class_=re.compile(r"(content|document|text|lege)", re.IGNORECASE))
        or soup.find("body")
    )

    if not content:
        return ""

    # get_text with separator="\n" means each HTML block element
    # becomes a separate line — this preserves article structure
    text = content.get_text(separator="\n")

    # Collapse 3+ consecutive newlines into just 2 (one blank line)
    # This keeps paragraph separation but removes excessive whitespace
    text = re.sub(r"\n{3,}", "\n\n", text)

    return text.strip()


# ── Step 4: Split text into articles ──────────────────────────────────────────

def split_into_articles(raw_text: str) -> list[dict]:
    """

    Cut the full law text into individual articles.



    Romanian laws use these article header formats:

        "Articolul 1"         — standard numbered article

        "Articolul 142"       — larger numbered article

        "Articolul UNIC"      — law with only one article

        "+ Articolul 5"       — the site sometimes prepends a "+" character

        "Art. 5"              — some older laws use abbreviation

        "ART. 5"              — uppercase abbreviation



    The strategy:

      1. Find every article header using a regex

      2. Each header's text ends where the NEXT header begins

      3. Return a list of {"number": ..., "text": ...} dicts



    Edge cases handled:

      - No articles found → return the whole text as a single chunk

      - Empty article body (abrogated articles) → skip them

      - Roman numerals → handled up to common values (I through XXX)

    """
    # This regex matches article headers ONLY when they appear as standalone headers,
    # NOT when they appear inline inside body text like "prevederile art. 29 din Legea".
    #
    # The key insight: real article headers on this portal are ALWAYS:
    #   1. At the start of a line (^ with MULTILINE)
    #   2. Optionally preceded by a "+" character (the portal's expand widget)
    #   3. Followed by a number/identifier, then a newline or end of string
    #      — they are NOT followed by more inline text like "din Legea nr. X"
    #
    # So "art. 29 din Legea nr. 47/1992" is NOT a header — it has text after it.
    # But "+ Articolul 29\n" IS a header — it's standalone on its own line.
    #
    # The negative lookahead (?!\s+din\b|\s+alin\b|\s+lit\b) prevents matching
    # inline law cross-references like "art. 29 din Legea" or "art. 5 alin. (1)".
    article_pattern = re.compile(
        r"^[\+\s]*"                           # optional leading + or whitespace
        r"((?:Articolul|ARTICOLUL)\s+"        # must use full word "Articolul"
        r"(\d+(?:\^\d+)?|UNIC|[IVXLC]{1,6}))"  # number, UNIC, or roman numeral
        r"(?!\s+(?:din\b|alin\b|lit\b|pct\b|teza\b))",  # NOT an inline ref
        re.MULTILINE,
    )

    matches = list(article_pattern.finditer(raw_text))

    # Edge case: law has no article markers (rare, but happens with very old laws)
    if not matches:
        return [{"number": "Articolul 1", "text": raw_text.strip()}]

    articles = []
    for i, match in enumerate(matches):
        # group(1) = full header text e.g. "Articolul 29"
        header = match.group(1).strip()

        # The article text starts right after the header
        text_start = match.end()

        # The article text ends where the next article header begins
        # For the last article, it extends to the end of the document
        text_end = matches[i + 1].start() if i + 1 < len(matches) else len(raw_text)

        body = raw_text[text_start:text_end].strip()

        # Skip completely empty articles (these are abrogated/repealed ones
        # where the site only shows the header with nothing after it)
        if not body:
            continue

        # Also skip articles that are ONLY the word "Abrogat" or similar
        body_single_line = " ".join(body.split())
        if re.match(r"^[\(\[\*\s]*[Aa]brogat[\)\]\*\s\.]*$", body_single_line):
            continue

        articles.append({
            "number": header,
            "text":   body,
        })

    return articles


# ── Main entry point ──────────────────────────────────────────────────────────

def scrape_law(law_id: int) -> dict | None:
    """

    Full pipeline for one law ID:

      1. Download the HTML page (tries both URL patterns automatically)

      2. Extract the title

      3. Extract the raw plain text

      4. Split into individual articles

      5. Return a structured dict



    Returns None if the page couldn't be fetched or was empty.

    """
    print(f"\n  → Fetching ID {law_id}...")

    soup, url = fetch_law_page(law_id)
    if soup is None:
        return None

    raw_text = extract_raw_text(soup)

    if not raw_text or len(raw_text) < 200:
        print(f"  ✗ Skipping ID {law_id} — page looks empty after text extraction.")
        return None

    title    = extract_title(soup)
    articles = split_into_articles(raw_text)

    print(f"  ✓ '{title}' — {len(articles)} articles found.")
    print(f"    URL: {url}")

    return {
        "id":            law_id,
        "title":         title,
        "url":           url,
        "article_count": len(articles),
        "articles":      articles,
        "raw_text":      raw_text,
    }


# ── Quick test ────────────────────────────────────────────────────────────────
# Run this file directly to test a single law:
#   python html_scraper.py
if __name__ == "__main__":
    # Codul Muncii (republicat 2011) — verified ID from legislatie.just.ro
    result = scrape_law(128646)

    if result:
        print(f"\n{'='*60}")
        print(f"Title:    {result['title']}")
        print(f"URL:      {result['url']}")
        print(f"Articles: {result['article_count']}")
        print(f"\nFirst 3 articles:")
        for art in result["articles"][:3]:
            print(f"\n  [{art['number']}]")
            print(f"  {art['text'][:200]}...")
    else:
        print("Failed to scrape. Check your internet connection.")