src/trigger.py

"""

trigger.py — push & run the scraper kernel on Kaggle from the dashboard.



Fire-and-forget: `kaggle kernels push` uploads a generated runner script and

Kaggle executes it immediately on their servers. The kernel pulls the project

code from the attached Kaggle Dataset (gmaps-scraper-code), scrapes, scores,

and pushes leads.csv to the HF Dataset repo. The dashboard never waits for it.



Credentials (Space settings):

    KAGGLE_USERNAME  variable — kaggle.com username

    KAGGLE_KEY       secret   — API key from kaggle.com/settings -> Create New Token

    HF_TOKEN         secret   — must be a WRITE token (the kernel uses it to upload)



Security note: the HF token is embedded in the generated kernel source, so the

kernel MUST stay private (is_private is forced to true below).

"""

import json
import os
import re
import subprocess
import tempfile
from pathlib import Path

KERNEL_SLUG = "gmaps-scraper-auto"      # kernel created/updated by the trigger
CODE_DATASET = "gmaps-scraper-code"     # Kaggle Dataset holding the project .py files

RUNNER_TEMPLATE = '''"""Auto-generated by the dashboard trigger. Do not edit by hand."""

import glob

import os

import shutil

import subprocess

import sys



os.environ["HF_TOKEN"] = "__HF_TOKEN__"

os.environ["HF_DATASET_REPO"] = "__HF_REPO__"



KEYWORD = "__KEYWORD__"

CITY = "__CITY__"

MAX_RESULTS = __MAX_RESULTS__

WITH_DETAILS = __WITH_DETAILS__

MIN_REVIEWS = __MIN_REVIEWS__

MIN_RATING = __MIN_RATING__

ONLY_NEW = __ONLY_NEW__

MAX_REVIEWS = __MAX_REVIEWS__

MAX_PHOTOS = __MAX_PHOTOS__





def sh(cmd, check=True):

    print(f"$ {cmd}")

    rc = subprocess.run(cmd, shell=True).returncode

    if check and rc != 0:

        sys.exit(rc)

    return rc





# 1. Copy the project code from the attached Kaggle Dataset (any mount layout)

REQUIRED = ("scraper.py", "process.py", "enrich.py", "push_to_hf.py")

for path in glob.glob("/kaggle/input/**/*.py", recursive=True):

    if os.path.basename(path) in REQUIRED:

        shutil.copy(path, ".")

        print(f"Copied {path}")



# 2. Install dependencies

sh("pip install -q playwright pandas requests beautifulsoup4 huggingface_hub")

sh("playwright install --with-deps chromium")



# 3. Scrape — never fatal: captcha/blocks leave a partial CSV behind

details = "--details" if WITH_DETAILS else ""

extra = ""

if WITH_DETAILS and MAX_REVIEWS:

    extra += f" --max-reviews {MAX_REVIEWS}"

if WITH_DETAILS and MAX_PHOTOS:

    extra += f" --max-photos {MAX_PHOTOS}"

sh(f'python scraper.py --keyword "{KEYWORD}" --city "{CITY}" '

   f"--max-results {MAX_RESULTS} --min-rating {MIN_RATING} "

   f"--min-reviews {MIN_REVIEWS} {details}{extra}", check=False)



# A killed run may only have the checkpoint file — promote it

if not os.path.exists("data/leads.csv") and os.path.exists("data/leads_partial.csv"):

    shutil.copy("data/leads_partial.csv", "data/leads.csv")

    print("Recovered leads from the partial checkpoint.")

if not os.path.exists("data/leads.csv"):

    print("No CSV produced (blocked before anything was scraped) — nothing to push.")

    sys.exit(0)



# 4. Score (+ optionally drop leads already in the HF dataset)

dedupe = "--dedupe-hf" if ONLY_NEW else ""

sh(f"python process.py --min-reviews {MIN_REVIEWS} --min-rating {MIN_RATING} {dedupe}")



# 5. Push whatever we got (an empty run is skipped, not an error)

sh(f'python push_to_hf.py --file data/leads_scored.csv --label "{KEYWORD} {CITY}"',

   check=False)

print("DONE — check the run list in the dashboard.")

'''


def _env(username: str, key: str) -> dict:
    """Provide Kaggle credentials via env vars AND ~/.kaggle/kaggle.json.



    Different kaggle CLI versions look in different places, so cover both.

    """
    env = os.environ.copy()
    env["KAGGLE_USERNAME"] = username
    env["KAGGLE_KEY"] = key

    cfg_dir = Path.home() / ".kaggle"
    cfg_file = cfg_dir / "kaggle.json"
    try:
        cfg_dir.mkdir(parents=True, exist_ok=True)
        cfg_file.write_text(json.dumps({"username": username, "key": key}),
                            encoding="utf-8")
        cfg_file.chmod(0o600)
    except OSError:
        pass  # read-only home: env vars alone still work on the pinned CLI

    return env


def _sanitize(text: str) -> str:
    """Keep kernel source injection-proof: strip quotes/backslashes/newlines."""
    return re.sub(r'["\'\\\n\r]', " ", text).strip()


def trigger_scrape(*, keyword: str, city: str, max_results: int,

                   with_details: bool, min_reviews: int,

                   hf_repo: str, hf_token: str,

                   kaggle_username: str, kaggle_key: str,

                   min_rating: float = 0.0, only_new: bool = False,

                   max_reviews: int = 0, max_photos: int = 0) -> str:
    """Generate the kernel folder and push it to Kaggle. Returns CLI output."""
    if not hf_token:
        raise ValueError("HF_TOKEN is missing — it must be a WRITE token.")

    runner = (RUNNER_TEMPLATE
              .replace("__HF_TOKEN__", hf_token)
              .replace("__HF_REPO__", _sanitize(hf_repo))
              .replace("__KEYWORD__", _sanitize(keyword))
              .replace("__CITY__", _sanitize(city))
              .replace("__MAX_RESULTS__", str(int(max_results)))
              .replace("__WITH_DETAILS__", str(bool(with_details)))
              .replace("__MIN_REVIEWS__", str(int(min_reviews)))
              .replace("__MIN_RATING__", str(float(min_rating)))
              .replace("__ONLY_NEW__", str(bool(only_new)))
              .replace("__MAX_REVIEWS__", str(int(max_reviews)))
              .replace("__MAX_PHOTOS__", str(int(max_photos))))

    metadata = {
        "id": f"{kaggle_username}/{KERNEL_SLUG}",
        "title": KERNEL_SLUG,
        "code_file": "runner.py",
        "language": "python",
        "kernel_type": "script",
        "is_private": True,               # the HF token lives in the source — keep private
        "enable_internet": True,          # required: Maps + HF upload
        "enable_gpu": False,
        "dataset_sources": [f"{kaggle_username}/{CODE_DATASET}"],
        "competition_sources": [],
        "kernel_sources": [],
    }

    with tempfile.TemporaryDirectory() as d:
        Path(d, "runner.py").write_text(runner, encoding="utf-8")
        Path(d, "kernel-metadata.json").write_text(
            json.dumps(metadata, indent=1), encoding="utf-8")

        result = subprocess.run(
            ["kaggle", "kernels", "push", "-p", d],
            capture_output=True, text=True, timeout=120,
            env=_env(kaggle_username, kaggle_key),
        )

    if result.returncode != 0:
        raise RuntimeError(result.stderr.strip() or result.stdout.strip())
    return result.stdout.strip()


def kernel_status(kaggle_username: str, kaggle_key: str) -> str:
    """Ask Kaggle for the current status of the triggered kernel."""
    result = subprocess.run(
        ["kaggle", "kernels", "status", f"{kaggle_username}/{KERNEL_SLUG}"],
        capture_output=True, text=True, timeout=60,
        env=_env(kaggle_username, kaggle_key),
    )
    return (result.stdout + result.stderr).strip()