src/trigger.py

"""
trigger.py — push & run the scraper kernel on Kaggle from the dashboard.

Fire-and-forget: `kaggle kernels push` uploads a generated runner script and
Kaggle executes it immediately on their servers. The kernel pulls the project
code from the attached Kaggle Dataset (gmaps-scraper-code), scrapes, scores,
and pushes leads.csv to the HF Dataset repo. The dashboard never waits for it.

Credentials (Space settings):
    KAGGLE_USERNAME  variable — kaggle.com username
    KAGGLE_KEY       secret   — API key from kaggle.com/settings -> Create New Token
    HF_TOKEN         secret   — must be a WRITE token (the kernel uses it to upload)

Security note: the HF token is embedded in the generated kernel source, so the
kernel MUST stay private (is_private is forced to true below).
"""

import json
import os
import re
import subprocess
import tempfile
from pathlib import Path

KERNEL_SLUG = "gmaps-scraper-auto"      # kernel created/updated by the trigger
CODE_DATASET = "gmaps-scraper-code"     # Kaggle Dataset holding the project .py files

RUNNER_TEMPLATE = '''"""Auto-generated by the dashboard trigger. Do not edit by hand."""
import glob
import os
import shutil
import subprocess
import sys

os.environ["HF_TOKEN"] = "__HF_TOKEN__"
os.environ["HF_DATASET_REPO"] = "__HF_REPO__"

KEYWORD = "__KEYWORD__"
CITY = "__CITY__"
MAX_RESULTS = __MAX_RESULTS__
WITH_DETAILS = __WITH_DETAILS__
MIN_REVIEWS = __MIN_REVIEWS__
MIN_RATING = __MIN_RATING__
ONLY_NEW = __ONLY_NEW__
MAX_REVIEWS = __MAX_REVIEWS__
MAX_PHOTOS = __MAX_PHOTOS__


def sh(cmd, check=True):
    print(f"$ {cmd}")
    rc = subprocess.run(cmd, shell=True).returncode
    if check and rc != 0:
        sys.exit(rc)
    return rc


# 1. Copy the project code from the attached Kaggle Dataset (any mount layout)
REQUIRED = ("scraper.py", "process.py", "enrich.py", "push_to_hf.py")
for path in glob.glob("/kaggle/input/**/*.py", recursive=True):
    if os.path.basename(path) in REQUIRED:
        shutil.copy(path, ".")
        print(f"Copied {path}")

# 2. Install dependencies
sh("pip install -q playwright pandas requests beautifulsoup4 huggingface_hub")
sh("playwright install --with-deps chromium")

# 3. Scrape — never fatal: captcha/blocks leave a partial CSV behind
details = "--details" if WITH_DETAILS else ""
extra = ""
if WITH_DETAILS and MAX_REVIEWS:
    extra += f" --max-reviews {MAX_REVIEWS}"
if WITH_DETAILS and MAX_PHOTOS:
    extra += f" --max-photos {MAX_PHOTOS}"
sh(f'python scraper.py --keyword "{KEYWORD}" --city "{CITY}" '
   f"--max-results {MAX_RESULTS} --min-rating {MIN_RATING} "
   f"--min-reviews {MIN_REVIEWS} {details}{extra}", check=False)

# A killed run may only have the checkpoint file — promote it
if not os.path.exists("data/leads.csv") and os.path.exists("data/leads_partial.csv"):
    shutil.copy("data/leads_partial.csv", "data/leads.csv")
    print("Recovered leads from the partial checkpoint.")
if not os.path.exists("data/leads.csv"):
    print("No CSV produced (blocked before anything was scraped) — nothing to push.")
    sys.exit(0)

# 4. Score (+ optionally drop leads already in the HF dataset)
dedupe = "--dedupe-hf" if ONLY_NEW else ""
sh(f"python process.py --min-reviews {MIN_REVIEWS} --min-rating {MIN_RATING} {dedupe}")

# 5. Push whatever we got (an empty run is skipped, not an error)
sh(f'python push_to_hf.py --file data/leads_scored.csv --label "{KEYWORD} {CITY}"',
   check=False)
print("DONE — check the run list in the dashboard.")
'''


def _env(username: str, key: str) -> dict:
    """Provide Kaggle credentials via env vars AND ~/.kaggle/kaggle.json.

    Different kaggle CLI versions look in different places, so cover both.
    """
    env = os.environ.copy()
    env["KAGGLE_USERNAME"] = username
    env["KAGGLE_KEY"] = key

    cfg_dir = Path.home() / ".kaggle"
    cfg_file = cfg_dir / "kaggle.json"
    try:
        cfg_dir.mkdir(parents=True, exist_ok=True)
        cfg_file.write_text(json.dumps({"username": username, "key": key}),
                            encoding="utf-8")
        cfg_file.chmod(0o600)
    except OSError:
        pass  # read-only home: env vars alone still work on the pinned CLI

    return env


def _sanitize(text: str) -> str:
    """Keep kernel source injection-proof: strip quotes/backslashes/newlines."""
    return re.sub(r'["\'\\\n\r]', " ", text).strip()


def trigger_scrape(*, keyword: str, city: str, max_results: int,
                   with_details: bool, min_reviews: int,
                   hf_repo: str, hf_token: str,
                   kaggle_username: str, kaggle_key: str,
                   min_rating: float = 0.0, only_new: bool = False,
                   max_reviews: int = 0, max_photos: int = 0) -> str:
    """Generate the kernel folder and push it to Kaggle. Returns CLI output."""
    if not hf_token:
        raise ValueError("HF_TOKEN is missing — it must be a WRITE token.")

    runner = (RUNNER_TEMPLATE
              .replace("__HF_TOKEN__", hf_token)
              .replace("__HF_REPO__", _sanitize(hf_repo))
              .replace("__KEYWORD__", _sanitize(keyword))
              .replace("__CITY__", _sanitize(city))
              .replace("__MAX_RESULTS__", str(int(max_results)))
              .replace("__WITH_DETAILS__", str(bool(with_details)))
              .replace("__MIN_REVIEWS__", str(int(min_reviews)))
              .replace("__MIN_RATING__", str(float(min_rating)))
              .replace("__ONLY_NEW__", str(bool(only_new)))
              .replace("__MAX_REVIEWS__", str(int(max_reviews)))
              .replace("__MAX_PHOTOS__", str(int(max_photos))))

    metadata = {
        "id": f"{kaggle_username}/{KERNEL_SLUG}",
        "title": KERNEL_SLUG,
        "code_file": "runner.py",
        "language": "python",
        "kernel_type": "script",
        "is_private": True,               # the HF token lives in the source — keep private
        "enable_internet": True,          # required: Maps + HF upload
        "enable_gpu": False,
        "dataset_sources": [f"{kaggle_username}/{CODE_DATASET}"],
        "competition_sources": [],
        "kernel_sources": [],
    }

    with tempfile.TemporaryDirectory() as d:
        Path(d, "runner.py").write_text(runner, encoding="utf-8")
        Path(d, "kernel-metadata.json").write_text(
            json.dumps(metadata, indent=1), encoding="utf-8")

        result = subprocess.run(
            ["kaggle", "kernels", "push", "-p", d],
            capture_output=True, text=True, timeout=120,
            env=_env(kaggle_username, kaggle_key),
        )

    if result.returncode != 0:
        raise RuntimeError(result.stderr.strip() or result.stdout.strip())
    return result.stdout.strip()


def kernel_status(kaggle_username: str, kaggle_key: str) -> str:
    """Ask Kaggle for the current status of the triggered kernel."""
    result = subprocess.run(
        ["kaggle", "kernels", "status", f"{kaggle_username}/{KERNEL_SLUG}"],
        capture_output=True, text=True, timeout=60,
        env=_env(kaggle_username, kaggle_key),
    )
    return (result.stdout + result.stderr).strip()