scraper/scheduler.py

# COST: ZERO OpenAI chat tokens.
# Scraping is plain HTTP + BeautifulSoup. Diff checking uses difflib (stdlib).
# is_content_stale() is checked first — if content is fresh the job returns
# immediately, skipping both the HTTP fetch and re-indexing.
# Re-indexing (ingest_scraped_content) calls text-embedding-3-small for new
# chunks only — one-time embedding cost, no chat LLM.

import logging

from apscheduler.schedulers.asyncio import AsyncIOScheduler

from ingestion import ingest_scraped_content
from scraper.content_store import (
    get_content_diff,
    is_content_stale,
    is_multi_page_stale,
    load_latest_content,
    save_all_pages,
    save_site_content,
)
from scraper.web_scraper import AGENTRAX_URL, scrape_all_pages, scrape_site

logger = logging.getLogger(__name__)

_scheduler: AsyncIOScheduler | None = None


async def refresh_agentrax_content() -> None:
    # ── Homepage (single-page store, backward compat) ────────────────────────
    if not is_content_stale():
        logger.info("Homepage content is fresh — skipping single-page refresh.")
    else:
        logger.info("Scraping homepage %s ...", AGENTRAX_URL)
        data = await scrape_site(AGENTRAX_URL)
        if "error" in data:
            logger.error("Homepage scrape failed: %s", data["error"])
        else:
            previous = load_latest_content()
            if previous:
                diff = get_content_diff(previous, data)
                if diff["is_changed"]:
                    logger.warning("HOMEPAGE UPDATED: %s", diff["summary_of_changes"])
            save_site_content(data)
            ingest_scraped_content(data)
            logger.info("Homepage refreshed and re-indexed.")

    # ── All pages (multi-page store) ─────────────────────────────────────────
    if not is_multi_page_stale():
        logger.info("Multi-page content is fresh — skipping full-site refresh.")
        return

    logger.info("Scraping all AgentRax pages...")
    pages = await scrape_all_pages()

    if not pages:
        logger.error("Full-site scrape returned no results.")
        return

    save_all_pages(pages)
    for page in pages:
        ingest_scraped_content(page)
    logger.info("Full site refreshed: %d page(s) scraped and indexed.", len(pages))


def start_scheduler() -> None:
    global _scheduler
    _scheduler = AsyncIOScheduler()
    _scheduler.add_job(
        refresh_agentrax_content,
        trigger="interval",
        minutes=60,
        id="refresh_agentrax",
        replace_existing=True,
    )
    _scheduler.start()
    logger.info("Scheduler started — refresh every 60 minutes.")

    import asyncio
    asyncio.get_event_loop().create_task(refresh_agentrax_content())


def stop_scheduler() -> None:
    global _scheduler
    if _scheduler and _scheduler.running:
        _scheduler.shutdown(wait=False)
        logger.info("Scheduler stopped.")