scripts/import_json_to_supabase.py

#!/usr/bin/env python3
"""Import existing notice JSON files into Supabase/Postgres."""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from crawling.supabase_store import ensure_schema, upsert_notices  # noqa: E402


def load_notices(path: Path) -> list[dict[str, Any]]:
    data = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(data, list):
        raise ValueError(f"{path} must contain a JSON array")

    notices = []
    for index, item in enumerate(data, start=1):
        if not isinstance(item, dict):
            raise ValueError(f"{path}:{index} is not a JSON object")
        if not item.get("url") or not item.get("title"):
            raise ValueError(f"{path}:{index} must include title and url")
        notices.append(item)
    return notices


def import_file(path: Path, batch_size: int) -> int:
    notices = load_notices(path)
    total = 0

    for start in range(0, len(notices), batch_size):
        batch = notices[start:start + batch_size]
        total += upsert_notices(batch)
        print(f"{path}: {min(start + batch_size, len(notices))}/{len(notices)} upserted")

    return total


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Import notice JSON files into the Supabase notices table.",
    )
    parser.add_argument("paths", nargs="+", type=Path, help="JSON files to import")
    parser.add_argument("--batch-size", type=int, default=100, help="Rows per DB batch")
    parser.add_argument("--skip-schema", action="store_true", help="Do not create/check DB schema")
    args = parser.parse_args()

    if args.batch_size < 1:
        raise ValueError("--batch-size must be at least 1")

    if not args.skip_schema:
        ensure_schema()

    total = 0
    for path in args.paths:
        total += import_file(path, args.batch_size)

    print(f"Done. Upserted {total} rows from {len(args.paths)} file(s).")


if __name__ == "__main__":
    main()