src/pino/scraper.py

from __future__ import annotations

"""
TGSC (The Good Scents Company) material scraper.

This module fetches raw material HTML pages from TGSC and extracts the
properties we need for realistic physics:
  - Boiling point (K)
  - Vapor pressure (mmHg -> Pa)
  - Odor description string

The parser is intentionally regex-driven against the page text because TGSC
pages are legacy hand-written HTML without stable semantic markup. Callers can
pass either a URL, a raw HTML string, or a Camoufox page object.
"""

import logging
import re
from typing import Any

try:
    from camoufox.sync_api import Camoufox
except Exception:  # pragma: no cover - optional dependency for stealth mode
    Camoufox = None

import requests
from bs4 import BeautifulSoup

logger = logging.getLogger("pino.scraper")

# Conversion factor: 1 mmHg = 133.3224 Pa (exact definition used in PINO).
MMHG_TO_PA = 133.3224

# Default room temperature for boiling-point conversions if only Celsius is given.
ROOM_TEMP_K = 298.15


def _kelvin_from_string(text: str) -> float | None:
    """Try to extract a temperature in Kelvin or Celsius from free text."""
    # Celsius: 123.45 °C / 123.45 C
    c_match = re.search(r"([\d\.]+)\s*°?C", text)
    if c_match:
        c = float(c_match.group(1))
        return c + 273.15
    # Kelvin: 123.45 K
    k_match = re.search(r"([\d\.]+)\s*K\b", text)
    if k_match:
        return float(k_match.group(1))
    return None


def _extract_boiling_point(page_text: str) -> float | None:
    """
    Extract boiling point from page text.

    TGSC usually labels this as "Boiling Point:" followed by a temperature range.
    We take the midpoint of the range.
    """
    # Look for a labelled boiling point line, e.g. "Boiling Point: 176.00 to 177.00 °C. @ 760.00 mm Hg"
    bp_match = re.search(
        r"Boiling Point:\s*([\d\.]+)\s*(?:to\s+([\d\.]+))?\s*°?C",
        page_text,
        re.IGNORECASE,
    )
    if bp_match:
        low = float(bp_match.group(1))
        high = float(bp_match.group(2)) if bp_match.group(2) else low
        return (low + high) / 2.0 + 273.15
    # Fallback: any phrase like "bp 123.45 °C".
    fallback = re.search(r"\bb\.?p\.?\s*([\d\.]+)\s*°?C", page_text, re.IGNORECASE)
    if fallback:
        return float(fallback.group(1)) + 273.15
    return None


def _extract_vapor_pressure(page_text: str) -> float | None:
    """
    Extract vapor pressure in mmHg and convert to Pascals.

    TGSC labels this as "Vapor Pressure: 0.003000 mmHg @ 25.00 °C. (est)".
    We always convert the mmHg value to Pa using MMHG_TO_PA.
    """
    vp_match = re.search(
        r"Vapor Pressure:\s*([\d\.]+)\s*mmHg", page_text, re.IGNORECASE
    )
    if vp_match:
        vp_mmhg = float(vp_match.group(1))
        return vp_mmhg * MMHG_TO_PA
    return None


def _extract_odor_description(page_text: str) -> str:
    """
    Extract the odor description keyword string.

    TGSC labels this as "Odor Description: ...". We capture the remainder of the
    line and strip the trailing attribution/reference text.
    """
    odor_match = re.search(
        r"Odor Description:\s*at\s+\d+(?:\.\d+)?\s*\%\.\s*([^\n\r]+)",
        page_text,
        re.IGNORECASE,
    )
    if odor_match:
        return odor_match.group(1).strip().split("  ")[0].strip()
    odor_match = re.search(
        r"Odor Description:\s*([^\n\r]+)", page_text, re.IGNORECASE
    )
    if odor_match:
        return odor_match.group(1).strip().split("  ")[0].strip()
    odor_match = re.search(r"Odor:\s*([^\n\r]+)", page_text, re.IGNORECASE)
    if odor_match:
        return odor_match.group(1).strip().split("  ")[0].strip()
    return ""


def parse_tgsc_html(html: str) -> dict[str, Any]:
    """Parse a TGSC raw material HTML string and return extracted properties."""
    soup = BeautifulSoup(html, "html.parser")
    page_text = soup.get_text(" ", strip=True)

    bp_k = _extract_boiling_point(page_text)
    vp_pa = _extract_vapor_pressure(page_text)
    odor_desc = _extract_odor_description(page_text)

    return {
        "boiling_point_k": bp_k,
        "vapor_pressure_pa": vp_pa,
        "odor_description": odor_desc,
    }


def parse_tgsc_page_requests(url: str, timeout: float = 30.0) -> dict[str, Any]:
    """
    Fetch a TGSC raw material URL with plain requests and parse its properties.
    """
    try:
        response = requests.get(
            url,
            headers={"User-Agent": "Mozilla/5.0"},
            timeout=timeout,
        )
        response.raise_for_status()
    except Exception as exc:
        logger.warning("Failed to fetch TGSC page %s: %s", url, exc)
        return {
            "boiling_point_k": None,
            "vapor_pressure_pa": None,
            "odor_description": "",
            "error": str(exc),
        }

    return parse_tgsc_html(response.text)


def parse_tgsc_page_camofox(url: str, browser: Any | None = None) -> dict[str, Any]:
    """
    Fetch a TGSC raw material URL using Camoufox for stealth and parse properties.

    If a browser instance is provided, it will be reused; otherwise a temporary
    Camoufox browser is created and closed.
    """
    if Camoufox is None:
        raise RuntimeError("Camoufox is not installed")

    close_browser = False
    if browser is None:
        browser = Camoufox(headless=True).__enter__()
        close_browser = True

    try:
        context = browser.new_context(no_viewport=True)
        page = context.new_page()
        page.goto(url, wait_until="networkidle")
        html = page.content()
        return parse_tgsc_html(html)
    except Exception as exc:
        logger.warning("Camoufox failed to fetch TGSC page %s: %s", url, exc)
        return {
            "boiling_point_k": None,
            "vapor_pressure_pa": None,
            "odor_description": "",
            "error": str(exc),
        }
    finally:
        if close_browser and browser is not None:
            browser.close()


def parse_tgsc_page(
    url: str,
    *,
    use_camofox: bool = True,
    timeout: float = 30.0,
    fallback_to_requests: bool = True,
) -> dict[str, Any]:
    """
    Fetch a TGSC raw material URL and parse its properties.

    Parameters
    ----------
    url: str
        TGSC data page URL (e.g., https://www.thegoodscentscompany.com/data/rw1007831.html).
    use_camofox: bool
        If True, use Camoufox for stealth; otherwise plain requests.
    timeout: float
        Request timeout for plain requests mode.
    fallback_to_requests: bool
        If True, retry with plain requests when Camoufox fails or returns no
        parseable TGSC data.

    Returns
    -------
    dict with boiling_point_k, vapor_pressure_pa, odor_description, and optionally error.
    """
    if use_camofox:
        try:
            result = parse_tgsc_page_camofox(url)
        except Exception as exc:
            if not fallback_to_requests:
                raise
            logger.warning("Camoufox unavailable for %s; falling back to requests: %s", url, exc)
            result = {
                "boiling_point_k": None,
                "vapor_pressure_pa": None,
                "odor_description": "",
                "error": str(exc),
            }

        has_data = any(
            result.get(key)
            for key in ("boiling_point_k", "vapor_pressure_pa", "odor_description")
        )
        if has_data or not fallback_to_requests:
            return result

        fallback = parse_tgsc_page_requests(url, timeout=timeout)
        if result.get("error") and "error" not in fallback:
            fallback["camofox_error"] = result["error"]
        return fallback
    return parse_tgsc_page_requests(url, timeout=timeout)


def mmhg_to_pa(mmhg: float) -> float:
    """Public helper for the strict mmHg -> Pa conversion."""
    return mmhg * MMHG_TO_PA


def search_tgsc(
    query: str,
    browser: Any | None = None,
) -> list[dict[str, str]]:
    """
    Search TGSC for materials by name and return candidate data page URLs.

    Returns a list of {"name": str, "url": str} records.
    """
    if Camoufox is None:
        raise RuntimeError("Camoufox is not installed")

    close_browser = False
    if browser is None:
        browser = Camoufox(headless=True).__enter__()
        close_browser = True

    results: list[dict[str, str]] = []
    try:
        context = browser.new_context(no_viewport=True)
        page = context.new_page()
        page.goto("https://www.thegoodscentscompany.com/search.php", wait_until="networkidle")
        page.fill('input[name="qName"]', query)
        page.click('input[type="Image"]')
        page.wait_for_load_state("networkidle")
        html = page.content()
        soup = BeautifulSoup(html, "html.parser")
        seen: set[str] = set()
        for tr in soup.find_all("tr"):
            a = tr.find("a", onclick=True)
            if a and "openMainWindow" in a.get("onclick", ""):
                m = re.search(r"openMainWindow\('([^']+)'\)", a["onclick"])
                if not m:
                    continue
                path = m.group(1)
                if path.startswith("/"):
                    url = f"https://www.thegoodscentscompany.com{path}"
                else:
                    url = f"https://www.thegoodscentscompany.com/{path}"
                if url in seen:
                    continue
                seen.add(url)
                name = a.get_text(strip=True)
                prefix = ""
                prefix_td = tr.find("td", class_="lstw9")
                if prefix_td:
                    prefix = prefix_td.get_text(strip=True)
                results.append({"name": f"{prefix} {name}".strip(), "url": url})
    finally:
        if close_browser and browser is not None:
            browser.close()

    return results


if __name__ == "__main__":
    import sys

    if len(sys.argv) < 2:
        print("Usage: python -m pino.scraper <tgsc-url>")
        sys.exit(1)

    result = parse_tgsc_page(sys.argv[1])
    print(result)