backend/cron.py

import time
import asyncio
from datetime import datetime, timedelta

from backend.config import PUBLIC_BOARDS, CRON_QUERIES
from backend.database import save_global_jobs, deactivate_old_jobs, get_global_stats

from backend.scrapers.nigeria import NigerianJobScraper
from backend.scrapers.ngos import NGOJobScraper
from backend.scrapers.international import InternationalJobScraper
from backend.scrapers.highimpact import HighImpactScraper
from backend.scrapers.web3 import Web3JobScraper
from backend.scrapers.ats import ATSJobScraper

_MAX_JOBS = 1000
_BOARD_TIMEOUT = 45  # seconds max per board (all queries combined)

SCRAPER_MAP = {
    "nigeria": NigerianJobScraper(),
    "ngo": NGOJobScraper(),
    "international": InternationalJobScraper(),
    "highimpact": HighImpactScraper(),
    "web3": Web3JobScraper(),
    "ats": ATSJobScraper(),
}


def _scrape_board(board_name: str, cat: str, query: str) -> list[dict]:
    scraper = SCRAPER_MAP.get(cat)
    if not scraper:
        return []
    method_name = f"scrape_{board_name}"
    method = getattr(scraper, method_name, None)
    if not method:
        return []
    try:
        if asyncio.iscoroutinefunction(method):
            return []
        result = method(query)
        if isinstance(result, list):
            for j in result:
                j["source"] = board_name
                j["category"] = cat
            return result
    except Exception as e:
        print(f"[cron] Error scraping {board_name} ({cat}/{query}): {e}")
    return []


def run_nightly_scrape() -> dict:
    start = time.time()
    all_jobs = []
    queries = CRON_QUERIES

    print(f"[cron] Nightly scrape starting at {datetime.now().isoformat()}")

    board_yields = {}
    for cat, name, cfg in PUBLIC_BOARDS:
        if not cfg.get("enabled", True):
            continue
        board_got_jobs = False
        board_start = time.time()
        for query in queries:
            if len(all_jobs) >= _MAX_JOBS * 2:
                break
            if time.time() - board_start > _BOARD_TIMEOUT:
                print(f"[cron] Timeout for {name} ({cat}) after {_BOARD_TIMEOUT}s")
                break
            jobs = _scrape_board(name, cat, query)
            if jobs:
                all_jobs.extend(jobs)
                board_got_jobs = True
                board_yields[name] = board_yields.get(name, 0) + len(jobs)
                if board_got_jobs:
                    break

    # Dedup by URL
    seen = set()
    unique = []
    for j in all_jobs:
        u = j.get("url", "")
        if u and u not in seen:
            seen.add(u)
            unique.append(j)
            if len(unique) >= _MAX_JOBS:
                break

    # Save to global pool
    saved = save_global_jobs(unique)

    # Log per-board yield
    print(f"[cron] Board yields:")
    for bname, cnt in sorted(board_yields.items(), key=lambda x: -x[1]):
        print(f"  {bname}: {cnt}")

    # Clean old jobs
    removed = deactivate_old_jobs(7)

    elapsed = time.time() - start
    result = {
        "scraped": len(all_jobs),
        "unique": len(unique),
        "saved": saved,
        "removed_old": removed,
        "elapsed_seconds": round(elapsed, 1),
        "timestamp": datetime.now().isoformat(),
    }
    print(f"[cron] Done: {result}")
    return result


def setup_scheduler(app):
    try:
        from apscheduler.schedulers.background import BackgroundScheduler
        from apscheduler.triggers.cron import CronTrigger

        scheduler = BackgroundScheduler()

        # Run at 6:00 AM daily
        scheduler.add_job(
            run_nightly_scrape,
            CronTrigger(hour=6, minute=0),
            id="morning_scrape",
            replace_existing=True,
        )

        # Run at 3:00 PM daily
        scheduler.add_job(
            run_nightly_scrape,
            CronTrigger(hour=15, minute=0),
            id="afternoon_scrape",
            replace_existing=True,
        )

        # Also run once on startup (after 60s delay — lets the server warm up)
        scheduler.add_job(
            run_nightly_scrape,
            trigger="date",
            run_date=datetime.now() + timedelta(seconds=60),
            id="startup_scrape",
            replace_existing=True,
            max_instances=1,
        )

        scheduler.start()
        print("[cron] Scheduler started. Scrapes at 6:00 AM and 3:00 PM daily.")
        return scheduler
    except ImportError:
        print("[cron] APScheduler not installed. Install with: pip install apscheduler")
        return None
    except Exception as e:
        print(f"[cron] Failed to start scheduler: {e}")
        return None