tools/scraping.py

import time, os, argparse

from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger

from src.config import config
from src.pipeline.pipeline import ImportPipeline
from src.scraping.scraper import Scraper
from src.utils.logging import init_logging, get_logger
from src.utils.tools import call_with_exponential_backoff
from src.notification.notification_center import NotificationCenter


def scraping_task(full_scrape: bool):
    init_logging()
    logger = get_logger('scraper.scheduler')
    def scrape():
        logger.info("Initiating scraping task")
        try:
            scraper = Scraper(scrape_all=full_scrape)
            pipeline = ImportPipeline()

            for target_url in config.scraping.TARGET_URLS:
                manifest = scraper.scrape_target(target_url)
                if not manifest:
                    continue
                pipeline.import_from_scraper(manifest)
                scraper.commit_scrape(manifest)

            logger.info("Scraper task finished gracefully")
        except Exception as e:
            logger.error(f"Scraping task was interrupted: {e}")
            raise e

    result = call_with_exponential_backoff(scrape)

    if result['status'] != 'OK':
        center = NotificationCenter()
        center.send_notification(
            "⛔ ERROR: Scraping failed",
            f"Scraping procedure failed after {config.scraping.MAX_RETRIES} attempts with message: {result['last_error']}",     
            "all",
            [
                os.path.join(config.paths.LOGS, 'scraping.log')
            ]
        )
        raise result['last_error']


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--init_sched', action='store_true')
    parser.add_argument('--full_scrape', action='store_true')
    return parser.parse_args()


def run_scheduler():
    scheduler = BackgroundScheduler()

    # Daily at 3 AM (Mon–Sat)
    scheduler.add_job(
        scraping_task,
        trigger=CronTrigger(day_of_week='mon-sat', hour=3, minute=0),
        args=[False],
        id='daily_scrape',
        max_instances=1,   
        coalesce=True,
        misfire_grace_time=3600,
    )

    # Sunday at 2 AM (full scrape)
    scheduler.add_job(
        scraping_task,
        trigger=CronTrigger(day_of_week='sun', hour=2, minute=0),
        args=[True],
        id='weekly_full_scrape',
        max_instances=1,
        coalesce=True,
        misfire_grace_time=3600,
    )

    scheduler.start()
    print("Scheduler started")

    try:
        while True:
            time.sleep(3600)
    except (KeyboardInterrupt, SystemExit):
        scheduler.shutdown()
        print("Scheduler stopped")


if __name__ == "__main__":
    args = parse_args()

    if args.init_sched:
        run_scheduler()
    else:
        scraping_task(args.full_scrape)