""" trigger.py — push & run the scraper kernel on Kaggle from the dashboard. Fire-and-forget: `kaggle kernels push` uploads a generated runner script and Kaggle executes it immediately on their servers. The kernel pulls the project code from the attached Kaggle Dataset (gmaps-scraper-code), scrapes, scores, and pushes leads.csv to the HF Dataset repo. The dashboard never waits for it. Credentials (Space settings): KAGGLE_USERNAME variable — kaggle.com username KAGGLE_KEY secret — API key from kaggle.com/settings -> Create New Token HF_TOKEN secret — must be a WRITE token (the kernel uses it to upload) Security note: the HF token is embedded in the generated kernel source, so the kernel MUST stay private (is_private is forced to true below). """ import json import os import re import subprocess import tempfile from pathlib import Path KERNEL_SLUG = "gmaps-scraper-auto" # kernel created/updated by the trigger CODE_DATASET = "gmaps-scraper-code" # Kaggle Dataset holding the project .py files RUNNER_TEMPLATE = '''"""Auto-generated by the dashboard trigger. Do not edit by hand.""" import glob import os import shutil import subprocess import sys os.environ["HF_TOKEN"] = "__HF_TOKEN__" os.environ["HF_DATASET_REPO"] = "__HF_REPO__" KEYWORD = "__KEYWORD__" CITY = "__CITY__" MAX_RESULTS = __MAX_RESULTS__ WITH_DETAILS = __WITH_DETAILS__ MIN_REVIEWS = __MIN_REVIEWS__ MIN_RATING = __MIN_RATING__ ONLY_NEW = __ONLY_NEW__ MAX_REVIEWS = __MAX_REVIEWS__ MAX_PHOTOS = __MAX_PHOTOS__ def sh(cmd, check=True): print(f"$ {cmd}") rc = subprocess.run(cmd, shell=True).returncode if check and rc != 0: sys.exit(rc) return rc # 1. Copy the project code from the attached Kaggle Dataset (any mount layout) REQUIRED = ("scraper.py", "process.py", "enrich.py", "push_to_hf.py") for path in glob.glob("/kaggle/input/**/*.py", recursive=True): if os.path.basename(path) in REQUIRED: shutil.copy(path, ".") print(f"Copied {path}") # 2. Install dependencies sh("pip install -q playwright pandas requests beautifulsoup4 huggingface_hub") sh("playwright install --with-deps chromium") # 3. Scrape — never fatal: captcha/blocks leave a partial CSV behind details = "--details" if WITH_DETAILS else "" extra = "" if WITH_DETAILS and MAX_REVIEWS: extra += f" --max-reviews {MAX_REVIEWS}" if WITH_DETAILS and MAX_PHOTOS: extra += f" --max-photos {MAX_PHOTOS}" sh(f'python scraper.py --keyword "{KEYWORD}" --city "{CITY}" ' f"--max-results {MAX_RESULTS} --min-rating {MIN_RATING} " f"--min-reviews {MIN_REVIEWS} {details}{extra}", check=False) # A killed run may only have the checkpoint file — promote it if not os.path.exists("data/leads.csv") and os.path.exists("data/leads_partial.csv"): shutil.copy("data/leads_partial.csv", "data/leads.csv") print("Recovered leads from the partial checkpoint.") if not os.path.exists("data/leads.csv"): print("No CSV produced (blocked before anything was scraped) — nothing to push.") sys.exit(0) # 4. Score (+ optionally drop leads already in the HF dataset) dedupe = "--dedupe-hf" if ONLY_NEW else "" sh(f"python process.py --min-reviews {MIN_REVIEWS} --min-rating {MIN_RATING} {dedupe}") # 5. Push whatever we got (an empty run is skipped, not an error) sh(f'python push_to_hf.py --file data/leads_scored.csv --label "{KEYWORD} {CITY}"', check=False) print("DONE — check the run list in the dashboard.") ''' def _env(username: str, key: str) -> dict: """Provide Kaggle credentials via env vars AND ~/.kaggle/kaggle.json. Different kaggle CLI versions look in different places, so cover both. """ env = os.environ.copy() env["KAGGLE_USERNAME"] = username env["KAGGLE_KEY"] = key cfg_dir = Path.home() / ".kaggle" cfg_file = cfg_dir / "kaggle.json" try: cfg_dir.mkdir(parents=True, exist_ok=True) cfg_file.write_text(json.dumps({"username": username, "key": key}), encoding="utf-8") cfg_file.chmod(0o600) except OSError: pass # read-only home: env vars alone still work on the pinned CLI return env def _sanitize(text: str) -> str: """Keep kernel source injection-proof: strip quotes/backslashes/newlines.""" return re.sub(r'["\'\\\n\r]', " ", text).strip() def trigger_scrape(*, keyword: str, city: str, max_results: int, with_details: bool, min_reviews: int, hf_repo: str, hf_token: str, kaggle_username: str, kaggle_key: str, min_rating: float = 0.0, only_new: bool = False, max_reviews: int = 0, max_photos: int = 0) -> str: """Generate the kernel folder and push it to Kaggle. Returns CLI output.""" if not hf_token: raise ValueError("HF_TOKEN is missing — it must be a WRITE token.") runner = (RUNNER_TEMPLATE .replace("__HF_TOKEN__", hf_token) .replace("__HF_REPO__", _sanitize(hf_repo)) .replace("__KEYWORD__", _sanitize(keyword)) .replace("__CITY__", _sanitize(city)) .replace("__MAX_RESULTS__", str(int(max_results))) .replace("__WITH_DETAILS__", str(bool(with_details))) .replace("__MIN_REVIEWS__", str(int(min_reviews))) .replace("__MIN_RATING__", str(float(min_rating))) .replace("__ONLY_NEW__", str(bool(only_new))) .replace("__MAX_REVIEWS__", str(int(max_reviews))) .replace("__MAX_PHOTOS__", str(int(max_photos)))) metadata = { "id": f"{kaggle_username}/{KERNEL_SLUG}", "title": KERNEL_SLUG, "code_file": "runner.py", "language": "python", "kernel_type": "script", "is_private": True, # the HF token lives in the source — keep private "enable_internet": True, # required: Maps + HF upload "enable_gpu": False, "dataset_sources": [f"{kaggle_username}/{CODE_DATASET}"], "competition_sources": [], "kernel_sources": [], } with tempfile.TemporaryDirectory() as d: Path(d, "runner.py").write_text(runner, encoding="utf-8") Path(d, "kernel-metadata.json").write_text( json.dumps(metadata, indent=1), encoding="utf-8") result = subprocess.run( ["kaggle", "kernels", "push", "-p", d], capture_output=True, text=True, timeout=120, env=_env(kaggle_username, kaggle_key), ) if result.returncode != 0: raise RuntimeError(result.stderr.strip() or result.stdout.strip()) return result.stdout.strip() def kernel_status(kaggle_username: str, kaggle_key: str) -> str: """Ask Kaggle for the current status of the triggered kernel.""" result = subprocess.run( ["kaggle", "kernels", "status", f"{kaggle_username}/{KERNEL_SLUG}"], capture_output=True, text=True, timeout=60, env=_env(kaggle_username, kaggle_key), ) return (result.stdout + result.stderr).strip()