src/crawler/scheduler.ts

import cron from 'node-cron';
import db from './db';
import { v4 as uuidv4 } from 'uuid';

export function initScheduler() {
  console.log('Initializing crawler scheduler...');

  // 1. Daily crawler for peer_bank and sdk_vendor
  cron.schedule('0 2 * * *', () => {
    console.log('Running daily crawler job (0 2 * * *)...');
    triggerJobs('daily');
  });

  // 2. 4h crawler for regulator
  cron.schedule('0 */4 * * *', () => {
    console.log('Running 4h crawler job (0 */4 * * *)...');
    triggerJobs('4h');
  });

  // 3. Retry logic (e.g. run every 10 minutes to find failed jobs to retry)
  cron.schedule('*/10 * * * *', () => {
    console.log('Running retry checker job...');
    handleRetries();
  });
}

function triggerJobs(frequency: string) {
  try {
    const sources = db.prepare('SELECT source_id FROM source_registry WHERE enabled = 1 AND crawl_frequency = ?').all(frequency) as { source_id: string }[];
    
    if (sources.length === 0) return;

    const now = new Date().toISOString();
    const stmt = db.prepare(`
      INSERT INTO crawl_job (
        job_id, source_id, trigger_type, status, started_at
      ) VALUES (
        @job_id, @source_id, @trigger_type, @status, @started_at
      )
    `);

    const insertMany = db.transaction((srcs: { source_id: string }[]) => {
      for (const src of srcs) {
        const job_id = `job_${uuidv4().replace(/-/g, '').substring(0, 16)}`;
        stmt.run({
          job_id,
          source_id: src.source_id,
          trigger_type: 'schedule',
          status: 'queued',
          started_at: now
        });
        
        // Here we would normally run the crawler job asynchronously
        console.log(`Job ${job_id} queued for source ${src.source_id}`);
      }
    });

    insertMany(sources);
    console.log(`Triggered ${sources.length} jobs for frequency ${frequency}`);
  } catch (error) {
    console.error('Failed to trigger jobs:', error);
  }
}

function handleRetries() {
  // Logic placeholders:
  // 1. Find jobs that failed and haven't reached max retries
  // 2. If retry_count < 1 (for 10 min retry), increment retry_count and requeue
  // 3. If consecutive failures > 3, mark source degraded
  // 4. If consecutive failures > 7 days, put into manual check queue
  console.log('Retry checker placeholder executed');
}