src/scraper/cleaner.py

"""

cleaner.py

──────────

Cleans raw Romanian legal text after scraping.



The problems we solve:

  1. Diacritic inconsistency

     Romanian has TWO Unicode encodings for ș and ț that look identical

     on screen but are DIFFERENT bytes. This breaks NLP models.

       - Correct modern form:  ș (U+015F),  ț (U+0163)

       - Legacy cedilla form:  ş (U+015E),  ţ (U+0162)  ← used on old RO sites

     We always normalize to the correct modern form.



  2. Encoding mojibake

     The portal sometimes serves pages with the wrong encoding header,

     which turns "ș" into garbage like "ÅŸ". The ftfy library fixes this.



  3. Site boilerplate

     Every page from the portal contains the same navigation text, browser

     upgrade warnings, copyright notices, etc. We strip all of it.



  4. Amendment noise

     Romanian laws are heavily amended. Many articles contain only a list

     of "modified by Law X, completed by OUG Y..." references with no

     actual legal text. These are useless for RAG and we filter them out.



  5. Abrogated articles

     Repealed articles only say "Abrogat" — we skip those.



  6. Footnote separators

     The portal adds "─────" lines followed by footnote text.

     We remove everything from the separator onwards.



Install: pip install ftfy

"""

import re
import unicodedata

# ftfy = "fixes text for you" — a library that repairs encoding problems
try:
    import ftfy
    HAS_FTFY = True
except ImportError:
    HAS_FTFY = False
    print("Tip: install ftfy for better encoding fixes: pip install ftfy")


# ── Diacritic normalization ───────────────────────────────────────────────────
# The two forms look the same in your editor but are different Unicode code points.
# Embeddings treat them as completely different characters — this MUST be fixed.
DIACRITIC_MAP = {
    "ş": "ș",       # U+015E → U+015F  (s with cedilla → s with comma below)
    "ţ": "ț",       # U+0162 → U+0163  (t with cedilla → t with comma below)
    "Ş": "Ș",       # uppercase versions
    "Ţ": "Ț",
    "\u015e": "Ș",  # explicit code point versions (same chars, just to be safe)
    "\u015f": "ș",
    "\u0162": "Ț",
    "\u0163": "ț",
}


# ── Boilerplate patterns ──────────────────────────────────────────────────────
# These strings appear on EVERY page from the portal — they are site UI,
# not law text. We strip them with regex.
_BOILERPLATE_PATTERNS = [
    # Browser upgrade warning banner
    r"A fost lansata versiunea Beta.*?corect\.",
    r"Datorita faptului ca folositi.*?corect!",
    r"De ce sa actualizez browserul\?.*?securitate\.",
    r"Browserele invechite.*?securitate\.",
    # Navigation links
    r"Reveniti in topul paginii",
    r"Forma printabilă",
    # Law metadata header (we already have this from the title)
    r"EMITENT\s*\n.*?\n",
    r"Publicat în\s*\n.*?\n",
    r"MONITORUL OFICIAL.*?\n",
    # Site identity strings
    r"Portal Legislativ",
    r"legislatie\.just\.ro",
    # Copyright footer
    r"©\s*\d{4}.*?rezervate\.",
    r"Conținutul acestui material.*?României\.",
]

# Compile all patterns into one big regex (compiled once = faster)
_BOILERPLATE_RE = re.compile(
    "|".join(_BOILERPLATE_PATTERNS),
    re.IGNORECASE | re.DOTALL,
)

# Lines to remove from article text (exact phrase matches)
_BLACKLIST_LINES = {
    "Reveniti in topul paginii",
    "Forma printabilă",
    "EMITENT",
    "PARLAMENTUL ROMÂNIEI",
    "PARLAMENTUL",
    "GUVERNUL",
    "GUVERNUL ROMÂNIEI",
    "PREȘEDINTELE ROMÂNIEI",
    "MONITORUL OFICIAL",
    "Portal Legislativ",
    "Pagina de start",
}

# Keywords that signal an article is ONLY an amendment reference list,
# not actual legal text. If more than half the lines contain these,
# the article is dropped.
_AMENDMENT_KEYWORDS = [
    "abrogată",
    "abrogat",
    "respinsă",
    "modificat prin",
    "completat prin",
    "înlocuit prin",
    "republicată",
]


# ── Individual cleaning functions ─────────────────────────────────────────────

def fix_encoding(text: str) -> str:
    """

    Fix garbled characters caused by wrong encoding detection.



    When a server says a page is UTF-8 but it's actually ISO-8859-2,

    characters get scrambled. For example:

        "ș" might appear as "Å£" or "ÅŸ"



    ftfy detects and repairs these patterns automatically.

    Without ftfy, we at least normalize to NFC (Unicode composed form).

    """
    if HAS_FTFY:
        return ftfy.fix_text(text)
    return unicodedata.normalize("NFC", text)


def fix_diacritics(text: str) -> str:
    """

    Replace legacy cedilla forms with the correct comma-below forms.



    This is the single most important cleaning step for Romanian NLP.

    Without it, the same word might be stored two different ways in

    your vector index, causing missed matches during retrieval.

    """
    for wrong, correct in DIACRITIC_MAP.items():
        text = text.replace(wrong, correct)
    return text


def remove_boilerplate(text: str) -> str:
    """Remove portal navigation/UI text from the full document."""
    return _BOILERPLATE_RE.sub("", text)


def normalize_whitespace(text: str) -> str:
    """

    Tidy up whitespace without destroying paragraph structure.



    What we do:

      - Replace tabs and non-breaking spaces (\\xa0) with regular spaces

      - Collapse 2+ spaces into one space

      - Collapse 3+ newlines into 2 (= one blank line between paragraphs)

      - Strip leading/trailing whitespace

    """
    text = text.replace("\t", " ").replace("\xa0", " ")
    text = re.sub(r" {2,}", " ", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def is_amendment_only(text: str) -> bool:
    """

    Return True if this article contains nothing but amendment references.



    An amendment-only article looks like:

        "Articolul 5 a fost modificat prin Legea 40/2011.

         Articolul 5 a fost completat prin OUG 53/2017."



    These tell us the law changed, but not what it says now.

    They're useless for a RAG system that needs the actual legal text.



    We check: if more than 50% of the non-empty lines contain amendment

    keywords, we consider the article as amendment-only and drop it.

    """
    lines = [l.strip() for l in text.split("\n") if l.strip()]
    if not lines:
        return True  # empty article = also drop it

    amendment_line_count = sum(
        1 for line in lines
        if any(keyword in line.lower() for keyword in _AMENDMENT_KEYWORDS)
    )

    return (amendment_line_count / len(lines)) > 0.5


def clean_article_text(text: str) -> str:
    """

    Clean the text of a single article.



    Steps performed:

      1. Remove blacklisted boilerplate lines

      2. Remove injected law title headers (the site sometimes repeats

         the law title inside each article's HTML section)

      3. Remove amendment history blocks at the top of an article

         (these start with "***) Note: ..." or similar footnote markers)

      4. Remove everything after the footnote separator line (─────)

      5. Normalize whitespace

    """
    # Step 1: Remove blacklisted whole-line phrases
    lines = text.split("\n")
    lines = [
        line for line in lines
        if not any(phrase in line for phrase in _BLACKLIST_LINES)
    ]
    text = "\n".join(lines)

    # Step 2: Remove law title headers that the portal injects
    # e.g. "LEGE nr. 53 din 28 iunie 2003\n" injected inside an article
    text = re.sub(
        r"LEGE\s+nr\.\s*\d+.*?(?=\n[A-ZĂÎȘȚ\(]|\Z)",
        "",
        text,
        flags=re.DOTALL | re.IGNORECASE,
    )
    text = re.sub(
        r"ORDONAN[ȚT][AĂ]\s+(?:DE\s+URGEN[ȚT][AĂ]\s+)?nr\.\s*\d+.*?(?=\n[A-ZĂÎȘȚ\(]|\Z)",
        "",
        text,
        flags=re.DOTALL | re.IGNORECASE,
    )

    # Step 3: Remove footnote marker blocks at the top
    # These look like: "**) NOTA: ..."
    text = re.sub(r"^\*+\).*?\n\n", "", text, flags=re.DOTALL)

    # Step 4: Remove footnote separator lines and everything after them
    # The portal adds "───────────────" lines before footnotes
    text = re.sub(r"[-─]{5,}.*", "", text, flags=re.DOTALL)

    # Step 5: Clean up whitespace
    return normalize_whitespace(text)


# ── Main cleaning function ────────────────────────────────────────────────────

def clean_law(law: dict) -> dict:
    """

    Apply all cleaning steps to a scraped law dictionary.



    Input (from html_scraper.scrape_law):

    {

        "id": 109567,

        "title": "LEGE 53 28/06/2003",

        "url": "...",

        "article_count": 298,

        "articles": [

            {"number": "Articolul 1", "text": "raw text..."},

            ...

        ],

        "raw_text": "full raw text..."

    }



    Output: same structure, but with cleaned text in every field.

    Articles that are empty, too short, or amendment-only are filtered out.

    """

    # ── Clean the title ───────────────────────────────────────────────────────
    title = re.sub(r"\s+", " ", law.get("title", "")).strip()

    # ── Clean the full raw text ───────────────────────────────────────────────
    raw = law.get("raw_text", "")
    raw = fix_encoding(raw)
    raw = fix_diacritics(raw)
    raw = remove_boilerplate(raw)
    raw = normalize_whitespace(raw)

    # ── Clean each article ────────────────────────────────────────────────────
    cleaned_articles = []

    for article in law.get("articles", []):
        text = article.get("text", "")

        # Apply encoding and diacritic fixes first
        text = fix_encoding(text)
        text = fix_diacritics(text)

        # Apply structural cleaning
        text = clean_article_text(text)

        # Skip articles that became too short after cleaning
        # (80 characters is roughly one short sentence — below that it's noise)
        if len(text) < 80:
            continue

        # Skip articles that only contain amendment history (no real legal text)
        if is_amendment_only(text):
            continue

        cleaned_articles.append({
            "number": article["number"],
            "text":   text,
        })

    print(f"  Cleaned: {len(law.get('articles', []))} articles → "
          f"{len(cleaned_articles)} kept after filtering.")

    return {
        **law,
        "title":         title,
        "raw_text":      raw,
        "articles":      cleaned_articles,
        "article_count": len(cleaned_articles),
    }


# ── Quick test ────────────────────────────────────────────────────────────────
if __name__ == "__main__":
    sample = {
        "id": 1,
        "title": "  Test Law  ",
        "url": "http://example.com",
        "raw_text": "Reveniti in topul paginii\nArticolul 1\nSalariaţii au dreptul la concediu.",
        "articles": [
            {
                "number": "Articolul 1",
                "text": (
                    "Salariaţii au dreptul la concediu de odihnă anual plătit.\n"
                    "─────────────────\n"
                    "Modificat prin Legea 40/2011."
                )
            },
            {
                "number": "Articolul 2",
                "text": "modificat prin Legea 1/2020\ncomplet prin OUG 2/2021"
            },
        ],
    }

    result = clean_law(sample)

    print(f"\nTitle: '{result['title']}'")
    print(f"Articles kept: {result['article_count']}")
    for art in result["articles"]:
        print(f"\n  [{art['number']}]")
        print(f"  '{art['text']}'")

    # Expected output:
    # Title: 'Test Law'
    # Articles kept: 1
    # [Articolul 1]
    # 'Salariații au dreptul la concediu de odihnă anual plătit.'
    # (Articolul 2 dropped — amendment only)
    # (footnote separator and everything after it removed from Articolul 1)