scripts/scrape_inaturalist.py

"""
Scrape Apis mellifera (Western honeybee) photos from the iNaturalist API.

Most observations are forager bees on flowers, not hive frames, but they
still give Moondream2 plenty of bee imagery to ground its responses, and
the high-quality bee close-ups are useful negatives / context for YOLO.

For hive-frame specific imagery (queens, varroa, swarm cells) see
data/SOURCES.md.

Usage:
    py scripts/scrape_inaturalist.py --pages 5
    py scripts/scrape_inaturalist.py --pages 20 --output data/raw/inaturalist

Resumable: re-running skips already-downloaded photos.
"""

from __future__ import annotations

import argparse
import json
import os
import time
from pathlib import Path
from urllib.parse import urlparse

import requests
from tqdm import tqdm


BASE_URL = "https://api.inaturalist.org/v1/observations"
USER_AGENT = "apiarist-hackathon/1.0 (build-small-hackathon)"


def search_page(taxon_name: str, page: int, per_page: int) -> dict:
    params = {
        "taxon_name": taxon_name,
        "photos": "true",
        "quality_grade": "research",
        "per_page": per_page,
        "page": page,
        "photo_license": "cc0,cc-by,cc-by-sa,cc-by-nc,cc-by-nd,cc-by-nc-sa,cc-by-nc-nd",
        "order_by": "votes",
    }
    resp = requests.get(
        BASE_URL,
        params=params,
        headers={"User-Agent": USER_AGENT},
        timeout=30,
    )
    resp.raise_for_status()
    return resp.json()


def upgrade_photo_url(url: str) -> str:
    """iNaturalist returns small thumbnails by default. Replace with medium."""
    return (
        url.replace("/square.", "/medium.")
        .replace("/small.", "/medium.")
    )


def download_photo(url: str, dest: Path) -> None:
    resp = requests.get(
        url, headers={"User-Agent": USER_AGENT}, timeout=30, stream=True
    )
    resp.raise_for_status()
    with open(dest, "wb") as f:
        for chunk in resp.iter_content(8192):
            f.write(chunk)


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=5)
    parser.add_argument("--per-page", type=int, default=100)
    parser.add_argument(
        "--output", type=str, default="data/raw/inaturalist"
    )
    parser.add_argument("--taxon", type=str, default="Apis mellifera")
    args = parser.parse_args()

    output_dir = Path(args.output)
    output_dir.mkdir(parents=True, exist_ok=True)
    metadata_path = output_dir / "metadata.jsonl"

    seen: set[int] = set()
    if metadata_path.exists():
        with open(metadata_path) as f:
            for line in f:
                try:
                    seen.add(json.loads(line)["photo_id"])
                except Exception:
                    pass
        print(f"Resume mode: {len(seen)} photos already downloaded.")

    total_new = 0

    with open(metadata_path, "a", encoding="utf-8") as meta_f:
        for page in range(1, args.pages + 1):
            print(f"\n=== Page {page}/{args.pages} ===")
            try:
                data = search_page(args.taxon, page, args.per_page)
            except Exception as e:
                print(f" [!] page {page} failed: {e}")
                time.sleep(2)
                continue

            obs_list = data.get("results", [])
            print(f" {len(obs_list)} observations returned")

            for obs in tqdm(obs_list, desc=f"page {page}", leave=False):
                for photo in obs.get("photos", []):
                    photo_id = photo.get("id")
                    if photo_id is None or photo_id in seen:
                        continue

                    raw_url = photo.get("url", "")
                    if not raw_url:
                        continue
                    url = upgrade_photo_url(raw_url)
                    ext = os.path.splitext(urlparse(url).path)[1] or ".jpg"
                    dest = output_dir / f"{photo_id}{ext}"

                    try:
                        download_photo(url, dest)
                    except Exception as e:
                        tqdm.write(f" [!] photo {photo_id} failed: {e}")
                        continue

                    rec = {
                        "photo_id": photo_id,
                        "url": url,
                        "license": photo.get("license_code"),
                        "observation_id": obs.get("id"),
                        "observed_on": obs.get("observed_on"),
                        "place_guess": obs.get("place_guess"),
                        "filename": dest.name,
                    }
                    meta_f.write(json.dumps(rec) + "\n")
                    meta_f.flush()
                    seen.add(photo_id)
                    total_new += 1

            # iNaturalist asks for max 1 req/sec, be polite
            time.sleep(1.5)

    print(f"\n Done. Downloaded {total_new} new photos this run.")
    print(f" Collection total: {len(seen)} photos in {output_dir}")


if __name__ == "__main__":
    main()