scraper.js

/**
 * Taiwan 今彩539 Lottery Scraper v2 - Uses the JSON API endpoint
 * Directly calls the internal JSON API to fetch all historical draw data
 */

const https = require('https');
const fs = require('fs');

// The site loads data via this JSON API - much cleaner than HTML parsing
const JSON_API = 'https://www.pilio.idv.tw/Json_ltonew.asp';

function httpsRequest(urlStr, method, postData) {
  return new Promise((resolve, reject) => {
    const url = new URL(urlStr);
    const body = postData || '';
    const opts = {
      hostname: url.hostname,
      port: 443,
      path: url.pathname + url.search,
      method: method || 'GET',
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        'Accept': 'application/json, text/javascript, */*',
        'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8',
        'Referer': 'https://www.pilio.idv.tw/lto539/list.asp',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Content-Length': Buffer.byteLength(body)
      }
    };

    const req = https.request(opts, (res) => {
      let data = '';
      res.setEncoding('utf8');
      res.on('data', chunk => { data += chunk; });
      res.on('end', () => resolve({ status: res.statusCode, body: data }));
    });
    req.on('error', reject);
    req.setTimeout(15000, () => { req.destroy(); reject(new Error('Request timeout')); });
    if (body) req.write(body);
    req.end();
  });
}

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

// Fetch a page of data using the JSON API
// Lindex: the index to fetch from (0 = oldest, use dex value from response to continue)
// Ldesc=0 means ascending (old to new), 1 means descending (new to old)
async function fetchJsonBatch(lindex, ldesc) {
  const postBody = `Lkind=lto539&Lindex=${lindex}&Ldesc=${ldesc}`;
  const res = await httpsRequest(JSON_API, 'POST', postBody);
  
  if (res.status !== 200) {
    throw new Error(`HTTP ${res.status}`);
  }
  
  try {
    return JSON.parse(res.body);
  } catch (e) {
    // Try to clean and parse
    const cleaned = res.body.trim();
    if (!cleaned || cleaned === 'null' || cleaned === '{}') return null;
    throw new Error(`JSON parse error: ${e.message}, body: ${res.body.slice(0, 200)}`);
  }
}

// The paginated HTML endpoint as fallback
async function fetchPageHtml(pageNum) {
  const url = `https://www.pilio.idv.tw/lto539/list.asp?indexpage=${pageNum}&orderby=old`;
  const res = await httpsRequest(url, 'GET');
  return res.body;
}

// Parse lottery numbers from the HTML-embedded script data
// The page embeds a script that renders a table with date and numbers
function parseHtmlPage(html) {
  const draws = [];
  
  // The data table is built via JS from a JSON object embedded in the page
  // Look for the embedded JSON data pattern: {"lotto":[{"date":"...","num":"...","dex":...},...]}
  const jsonMatch = html.match(/\{"lotto":\[[\s\S]*?\]\}/);
  if (jsonMatch) {
    try {
      const obj = JSON.parse(jsonMatch[0]);
      for (const item of obj.lotto) {
        const draw = parseLottoItem(item);
        if (draw) draws.push(draw);
      }
      return { draws, lastDex: obj.lotto.length > 0 ? obj.lotto[obj.lotto.length - 1].dex : null };
    } catch (e) {
      // Fall through to regex parsing
    }
  }
  
  // Fallback: look for the specific table structure with number cells
  // Pattern: the numbers column contains 5 two-digit numbers
  // The date column contains MM/DD format
  // Use a stricter pattern: look for cells that contain exactly 5 space-separated 2-digit numbers
  const numberCellPattern = /class="number-cell"[^>]*>([\d\s]+)<\/td>/gi;
  const dateCellPattern = /class="date-cell"[^>]*>([\s\S]*?)<\/td>/gi;
  
  const numberCells = [];
  const dateCells = [];
  
  let m;
  while ((m = numberCellPattern.exec(html)) !== null) {
    numberCells.push(m[1].trim());
  }
  while ((m = dateCellPattern.exec(html)) !== null) {
    dateCells.push(m[1].replace(/<[^>]+>/g, ' ').trim());
  }
  
  for (let i = 0; i < Math.min(numberCells.length, dateCells.length); i++) {
    // Parse 5 numbers from the number cell
    const nums = numberCells[i].match(/\d{1,2}/g) || [];
    const validNums = nums.filter(n => parseInt(n) >= 1 && parseInt(n) <= 39).map(n => String(parseInt(n)).padStart(2, '0'));
    
    if (validNums.length === 5) {
      // Parse date
      const dateText = dateCells[i].replace(/\s+/g, ' ').trim();
      draws.push({ date: dateText, numbers: validNums });
    }
  }
  
  return { draws, lastDex: null };
}

function parseLottoItem(item) {
  // item.date format: "01/01<br>07(一)" = MM/DD<br>YY(DayOfWeek)
  // item.num format: "09, 11, 27, 28, 38"
  // item.dex: serial index number

  if (!item.date || !item.num) return null;

  // Parse date: split on <br> tag
  const dateParts = item.date.split(/<br\s*\/?>/i);
  const mmdd = dateParts[0] ? dateParts[0].replace(/<[^>]+>/g, '').trim() : '';
  const yearDow = dateParts[1] ? dateParts[1].replace(/<[^>]+>/g, '').trim() : '';

  // Extract MM and DD
  const mmddMatch = mmdd.match(/(\d{1,2})[\/-](\d{2})/);
  const mm = mmddMatch ? mmddMatch[1].padStart(2, '0') : '??';
  const dd = mmddMatch ? mmddMatch[2] : '??';

  // Extract 2-digit year and day-of-week: e.g. "07(一)"
  const ydMatch = yearDow.match(/(\d{1,2})\(([^)]+)\)/);
  let fullYear = '????';
  let dow = '';
  if (ydMatch) {
    const yy = parseInt(ydMatch[1]);
    fullYear = yy < 50 ? `20${String(yy).padStart(2, '0')}` : `19${String(yy).padStart(2,'0')}`;
    dow = ydMatch[2];
  }

  const dateStr = `${fullYear}/${mm}/${dd}${dow ? ' (' + dow + ')' : ''}`;

  // Clean and parse numbers (comma-separated: "09, 11, 27, 28, 38")
  const numText = item.num.replace(/<[^>]+>/gi, '').replace(/&nbsp;/gi, ' ');
  const nums = numText.match(/\d{1,2}/g) || [];
  const validNums = nums
    .filter(n => parseInt(n) >= 1 && parseInt(n) <= 39)
    .map(n => String(parseInt(n)).padStart(2, '0'));

  if (validNums.length < 5) return null;

  return {
    date: dateStr,
    dateSort: `${fullYear}${mm}${dd}`, // for sorting
    numbers: validNums.slice(0, 5),
    dex: item.dex
  };
}

// ─── Try the JSON API approach ───────────────────────────────────────────────

async function testJsonApi() {
  console.log('Testing JSON API endpoint...');
  
  // Try with Lindex=0 (fetch from beginning)
  try {
    const result = await fetchJsonBatch(0, 0);
    if (result && result.lotto && result.lotto.length > 0) {
      console.log('JSON API works! Sample data:');
      console.log('Total in batch:', result.lotto.length);
      console.log('First item:', JSON.stringify(result.lotto[0]));
      console.log('Last item:', JSON.stringify(result.lotto[result.lotto.length - 1]));
      return { works: true, data: result };
    }
  } catch (e) {
    console.log('JSON API error:', e.message);
  }
  
  return { works: false };
}

async function main() {
  // First test the JSON API
  const test = await testJsonApi();
  
  if (test.works) {
    console.log('\nUsing JSON API to collect all data...');
    await collectViaJsonApi(test.data);
  } else {
    console.log('\nJSON API not available, falling back to HTML scraping with fixed parser...');
    await collectViaHtml();
  }
}

async function collectViaJsonApi(firstBatch) {
  const allDraws = [];
  
  // Process first batch
  for (const item of firstBatch.lotto) {
    const draw = parseLottoItem(item);
    if (draw) allDraws.push(draw);
  }
  
  let lastDex = firstBatch.lotto[firstBatch.lotto.length - 1].dex;
  let batchNum = 1;
  
  // Keep fetching batches until no more data
  while (true) {
    await sleep(300);
    batchNum++;
    process.stdout.write(`\r📥 Batch ${batchNum}, fetching after dex=${lastDex}... (${allDraws.length} draws)`);
    
    try {
      const batch = await fetchJsonBatch(lastDex, 0);
      
      if (!batch || !batch.lotto || batch.lotto.length === 0) {
        console.log('\n✅ No more data.');
        break;
      }
      
      let newCount = 0;
      for (const item of batch.lotto) {
        const draw = parseLottoItem(item);
        if (draw) { allDraws.push(draw); newCount++; }
      }
      
      if (newCount === 0) {
        console.log('\n✅ No new items in batch, done.');
        break;
      }
      
      lastDex = batch.lotto[batch.lotto.length - 1].dex;
    } catch (e) {
      console.log(`\n⚠️ Batch ${batchNum} failed: ${e.message}`);
      await sleep(1000);
    }
  }
  
  await finalize(allDraws);
}

async function collectViaHtml() {
  // Use the paginated HTML approach but with fixed parser
  // First, get total page count from page 1
  const page1 = await fetchPageHtml(1);
  const pageCountMatch = page1.match(/indexpage=(\d+)&orderby=old">\s*(?:最末頁|Last)/);
  const totalPages = pageCountMatch ? parseInt(pageCountMatch[1]) : 253;
  
  console.log(`📄 Total pages: ${totalPages}`);
  
  const allDraws = [];
  
  for (let p = 1; p <= totalPages; p++) {
    process.stdout.write(`\r📥 Page ${p}/${totalPages} (${allDraws.length} draws)`);
    try {
      const html = p === 1 ? page1 : await fetchPageHtml(p);
      const { draws } = parseHtmlPage(html);
      allDraws.push(...draws);
      await sleep(250);
    } catch (e) {
      console.log(`\n⚠️ Page ${p} failed: ${e.message}`);
    }
  }
  
  await finalize(allDraws);
}

// ─── Prediction Model ────────────────────────────────────────────────────────

function computeFrequency(draws) {
  const freq = {};
  for (let i = 1; i <= 39; i++) freq[String(i).padStart(2, '0')] = 0;
  for (const draw of draws) {
    for (const n of draw.numbers) {
      const key = String(parseInt(n)).padStart(2, '0');
      if (freq[key] !== undefined) freq[key]++;
    }
  }
  return freq;
}

function getLastSeenMap(draws) {
  const lastSeen = {};
  for (let i = 1; i <= 39; i++) lastSeen[String(i).padStart(2, '0')] = draws.length;
  for (let i = draws.length - 1; i >= 0; i--) {
    for (const num of draws[i].numbers) {
      const key = String(parseInt(num)).padStart(2, '0');
      if (lastSeen[key] === draws.length) lastSeen[key] = draws.length - 1 - i;
    }
  }
  return lastSeen;
}

// 5 buckets that evenly span 1–39
const SPREAD_BUCKETS = [
  [1,2,3,4,5,6,7,8],
  [9,10,11,12,13,14,15],
  [16,17,18,19,20,21,22],
  [23,24,25,26,27,28,29,30],
  [31,32,33,34,35,36,37,38,39],
];

function predictNextDraw(draws) {
  if (draws.length < 10) return [];

  // M3 SpreadOverdue: for each of 5 buckets, pick the most-overdue number
  // This guarantees spread across the full 1–39 range AND rewards overdue numbers
  const lastSeen = getLastSeenMap(draws);

  return SPREAD_BUCKETS.map(bucket => {
    return bucket
      .map(n => ({ n: String(n).padStart(2, '0'), gap: lastSeen[String(n).padStart(2, '0')] || 0 }))
      .sort((a, b) => b.gap - a.gap)[0].n;
  }).sort((a, b) => parseInt(a) - parseInt(b));
}

// ─── Markdown ─────────────────────────────────────────────────────────────────

function generateMarkdown(draws, prediction) {
  const freq = computeFrequency(draws);
  const lastSeen = getLastSeenMap(draws);
  const total = draws.length;
  
  const hot = Object.entries(freq).sort((a, b) => b[1] - a[1]).slice(0, 10);
  const cold = Object.entries(freq).sort((a, b) => a[1] - b[1]).slice(0, 10);
  const overdue = Object.entries(lastSeen).sort((a, b) => b[1] - a[1]).slice(0, 10);
  
  const lines = [
    '# 今彩539 歷史開獎號碼 & 預測分析',
    '',
    `> 資料來源：https://www.pilio.idv.tw/lto539/list.asp`,
    `> 共收錄 **${total}** 期開獎紀錄`,
    `> 產生時間：${new Date().toLocaleString('zh-TW')}`,
    '',
    '---',
    '',
    '## 🎯 下期預測號碼',
    '',
    '> 採用加權統計模型：熱號 40% + 遲到號 30% + 冷號 20% + 近期趨勢 10%',
    '>',
    '> ⚠️ 彩券具有不確定性，本預測僅供參考娛樂，請理性購買。',
    '',
    `### 預測號碼：**${prediction.join('  ·  ')}**`,
    '',
    '---',
    '',
    '## 📊 統計分析',
    '',
    '### 🔥 熱門號碼 Top 10',
    '',
    '| 號碼 | 出現次數 | 出現率 |',
    '|:---:|:---:|:---:|',
    ...hot.map(([num, count]) => `| **${num}** | ${count} | ${((count / total) * 100).toFixed(2)}% |`),
    '',
    '### 🧊 冷門號碼 Top 10',
    '',
    '| 號碼 | 出現次數 | 出現率 |',
    '|:---:|:---:|:---:|',
    ...cold.map(([num, count]) => `| **${num}** | ${count} | ${((count / total) * 100).toFixed(2)}% |`),
    '',
    '### ⏰ 最久未出現 (遲到號) Top 10',
    '',
    '| 號碼 | 距上次出現（期數）|',
    '|:---:|:---:|',
    ...overdue.map(([num, gap]) => `| **${num}** | ${gap === total ? '未曾出現' : gap} |`),
    '',
    '### 📈 全號碼出現頻率 (01–39)',
    '',
    '| 號碼 | 次數 | 出現率 | 號碼 | 次數 | 出現率 |',
    '|:---:|:---:|:---:|:---:|:---:|:---:|',
  ];
  
  const freqEntries = Object.entries(freq).sort((a, b) => parseInt(a[0]) - parseInt(b[0]));
  for (let i = 0; i < freqEntries.length; i += 2) {
    const [n1, c1] = freqEntries[i];
    const r1 = ((c1 / total) * 100).toFixed(2);
    if (i + 1 < freqEntries.length) {
      const [n2, c2] = freqEntries[i + 1];
      const r2 = ((c2 / total) * 100).toFixed(2);
      lines.push(`| ${n1} | ${c1} | ${r1}% | ${n2} | ${c2} | ${r2}% |`);
    } else {
      lines.push(`| ${n1} | ${c1} | ${r1}% | — | — | — |`);
    }
  }
  
  lines.push('', '---', '', '## 📋 歷史開獎紀錄（由新到舊）', '',
    '| # | 日期 | 號碼 1 | 號碼 2 | 號碼 3 | 號碼 4 | 號碼 5 |',
    '|:---:|:---:|:---:|:---:|:---:|:---:|:---:|');
  
  const reversed = [...draws].reverse();
  for (let i = 0; i < reversed.length; i++) {
    const draw = reversed[i];
    const idx = total - i;
    const [n1, n2, n3, n4, n5] = draw.numbers;
    lines.push(`| ${idx} | ${draw.date} | ${n1} | ${n2} | ${n3} | ${n4} | ${n5} |`);
  }
  
  return lines.join('\n');
}

async function finalize(allDraws) {
  console.log(`\n\n✅ Total draws collected: ${allDraws.length}`);
  
  // Dedup
  const seen = new Set();
  const unique = allDraws.filter(d => {
    const key = `${d.date}|${d.numbers.join(',')}`;
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  });
  
  console.log(`🔢 Unique draws: ${unique.length}`);
  
  // Validate: all draws should have exactly 5 unique numbers 1-39
  const valid = unique.filter(d => {
    if (d.numbers.length !== 5) return false;
    const nums = d.numbers.map(n => parseInt(n));
    const allValid = nums.every(n => n >= 1 && n <= 39);
    const allUnique = new Set(nums).size === 5;
    return allValid && allUnique;
  });
  
  console.log(`✔️  Valid draws (5 unique numbers 1-39): ${valid.length}`);
  
  if (valid.length < 100) {
    console.error('❌ Too few valid draws. Parsing may have failed.');
    console.log('Sample raw draws:');
    unique.slice(0, 5).forEach(d => console.log(d));
    process.exit(1);
  }
  
  console.log('\nSample draws (first 3):');
  valid.slice(0, 3).forEach(d => console.log(JSON.stringify(d)));
  console.log('Sample draws (last 3):');
  valid.slice(-3).forEach(d => console.log(JSON.stringify(d)));
  
  // Run prediction  
  console.log('\n🔮 Running prediction model...');
  const prediction = predictNextDraw(valid);
  console.log(`🎯 Predicted next draw: ${prediction.join(', ')}`);
  
  // Save markdown
  console.log('\n📝 Generating markdown...');
  const md = generateMarkdown(valid, prediction);
  fs.writeFileSync('lottery_539_history.md', md, 'utf8');
  console.log(`✅ Markdown saved: lottery_539_history.md (${(fs.statSync('lottery_539_history.md').size / 1024).toFixed(1)} KB)`);
  
  // Save JSON
  const jsonOut = { draws: valid, prediction, generatedAt: new Date().toISOString(), total: valid.length };
  fs.writeFileSync('lottery_539_data.json', JSON.stringify(jsonOut, null, 2), 'utf8');
  console.log(`✅ JSON saved: lottery_539_data.json`);
}

main().catch(console.error);