scripts/datasources/ma_pilot/mayor_boost.py

"""
Mayor-specific extraction helpers.

The default directory classifier (``classify_contact_directory_page``) only flags a page
as a contact directory when ``score >= 18``. A single-bio mayor page (one person, one
photo, no roster) often misses that threshold even though the row is exactly what we
want to capture. These helpers:

1. Recognize mayor-style URLs (``/mayor``, ``/mayors-office``, ``mayor_s_office/``, …).
2. Tag mayor-titled rows from any extractor pass so callers can persist them
   unconditionally instead of discarding low-score pages.
"""

from __future__ import annotations

import re
from typing import Any, Iterable
from urllib.parse import urlparse

_MAYOR_URL_RE = re.compile(
    r"(?:^|[/_-])"
    r"(?:mayors?\d*|mayor[-_]?s?[-_]?office|mayors?office|mayor_s_office)"
    r"(?:[/_.-]|$)",
    re.IGNORECASE,
)

_MAYOR_TITLE_RE = re.compile(r"\bmayor\b", re.IGNORECASE)


def is_mayor_seed_url(url: str) -> bool:
    """True when the URL path looks like a mayor's office / bio page."""
    if not url:
        return False
    try:
        path = (urlparse(url).path or "").lower()
    except Exception:
        return False
    if not path:
        return False
    return bool(_MAYOR_URL_RE.search(path))


def is_mayor_row(row: dict[str, Any]) -> bool:
    """True when a contact row has a Mayor-ish title.

    Excludes 'Vice Mayor' by itself by stripping that phrase before the match. A title
    like 'Mayor & Councilor-at-Large' or 'Acting Mayor' still matches.
    """
    title = str(row.get("title_or_role") or "").strip()
    if not title:
        return False
    stripped = re.sub(r"\bvice[\s-]?mayor\b", "", title, flags=re.IGNORECASE)
    return bool(_MAYOR_TITLE_RE.search(stripped))


def tag_mayor_rows(rows: Iterable[dict[str, Any]], *, source_page_url: str) -> list[dict[str, Any]]:
    """
    Return rows with ``is_mayor`` set to True for any row that should bypass the
    directory-score gate. Triggers when the source URL is a mayor page OR the row
    title contains "Mayor".
    """
    seed_is_mayor = is_mayor_seed_url(source_page_url)
    out: list[dict[str, Any]] = []
    for r in rows:
        tagged = dict(r)
        tagged["is_mayor"] = bool(is_mayor_row(r) or (seed_is_mayor and r.get("person_name")))
        out.append(tagged)
    return out