scripts/captureFixtures.js

const fs = require('fs')
const path = require('path')
const { createClient } = require('../utils/httpClient')
const { analyzePage } = require('../utils/pageAnalyzer')

const fixturesDir = path.join(__dirname, '..', '_debug', 'fixtures')
const servicesDir = path.join(__dirname, '..', 'services')
const candidatesPath = path.join(__dirname, '..', 'config', 'providerCandidates.json')

const client = createClient({
  headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
  timeout: 10000,
})

function extractBaseUrl(code) {
  const match = code.match(/BASE_URL\s*=\s*['"]([^'"]+)['"]/)
  return match ? match[1].replace(/\/$/, '') : null
}

function providerNameFromFile(filename) {
  return filename.replace(/Service\.js$/i, '').replace(/\.js$/i, '').toLowerCase()
}

async function probe(url, label) {
  try {
    const resp = await client.get(url, { validateStatus: () => true })
    return { url, status: resp.status, html: resp.data, ok: resp.status === 200 }
  } catch (err) {
    return { url, status: null, html: null, ok: false, error: err.message }
  }
}

async function captureFixtures() {
  if (!fs.existsSync(fixturesDir)) {
    fs.mkdirSync(fixturesDir, { recursive: true })
  }

  const candidates = fs.existsSync(candidatesPath)
    ? JSON.parse(fs.readFileSync(candidatesPath, 'utf-8'))
    : {}

  const serviceFiles = fs.readdirSync(servicesDir).filter(f => f.endsWith('.js'))
  const targets = []

  for (const file of serviceFiles) {
    const provider = providerNameFromFile(file)
    const code = fs.readFileSync(path.join(servicesDir, file), 'utf-8')
    const baseUrl = extractBaseUrl(code)
    if (baseUrl) {
      targets.push({ provider, url: baseUrl, source: 'service' })
    }
    const providerCandidates = candidates[provider] || []
    for (const domain of providerCandidates) {
      const candidateUrl = `https://${domain}`
      if (!targets.some(t => t.url === candidateUrl)) {
        targets.push({ provider, url: candidateUrl, source: 'candidate' })
      }
    }
  }

  console.log(`Total targets: ${targets.length}`)

  for (const target of targets) {
    const providerDir = path.join(fixturesDir, target.provider)
    if (!fs.existsSync(providerDir)) {
      fs.mkdirSync(providerDir, { recursive: true })
    }

    const result = await probe(target.url, `${target.provider}/${target.source}`)

    let pageType = 'unknown'
    let analysis = null
    if (result.html) {
      try {
        analysis = await analyzePage(result.html, result.url)
        pageType = analysis.pageType
      } catch {}
    }

    const statusLabel = result.status ? `http${result.status}` : 'error'
    const filename = `${pageType}_${statusLabel}_${target.source}_${target.provider}.html`
    const filepath = path.join(providerDir, filename)

    const meta = {
      capturedAt: new Date().toISOString(),
      url: target.url,
      provider: target.provider,
      source: target.source,
      status: result.status,
      pageType,
      analysis: analysis ? {
        pageType: analysis.pageType,
        botProtection: analysis.botProtection,
        contentHints: analysis.contentHints,
        linkPatterns: analysis.linkPatterns.slice(0, 3),
        hasRealContent: analysis.contentHints.hasRealContent,
      } : null,
      error: result.error || null,
    }

    const html = `<!--\n${JSON.stringify(meta, null, 2)}\n-->\n${result.html || ''}`
    fs.writeFileSync(filepath, html, 'utf-8')
    console.log(`[${result.status || 'ERR'}] ${pageType.padEnd(12)} ${target.url} → ${filename}`)
  }

  const summary = targets.map(t => {
    const dir = path.join(fixturesDir, t.provider)
    if (!fs.existsSync(dir)) return null
    const files = fs.readdirSync(dir).filter(f => f.includes(t.provider))
    return files.length > 0 ? `${t.provider}: ${files.length} fixtures` : null
  }).filter(Boolean)

  fs.writeFileSync(path.join(fixturesDir, '_summary.json'), JSON.stringify({
    capturedAt: new Date().toISOString(),
    totalTargets: targets.length,
    providers: summary,
  }, null, 2), 'utf-8')

  console.log(`\nDone. Fixtures in ${fixturesDir}`)
  console.log(summary.join('\n'))
}

captureFixtures().catch(err => {
  console.error('Fatal:', err.message)
  process.exit(1)
})