scraper649.js

/**
 * Taiwan 大樂透 (Lotto 6/49) Scraper - Uses the JSON API endpoint
 * Directly calls the internal JSON API to fetch all historical draw data
 * Numbers: 1-49, pick 6 regular numbers + 1 special number (特別號)
 * Draws: Monday and Thursday each week
 */

const https = require('https');
const fs = require('fs');

// The site loads data via this JSON API - much cleaner than HTML parsing
const JSON_API = 'https://www.pilio.idv.tw/Json_ltonew.asp';

function httpsRequest(urlStr, method, postData) {
  return new Promise((resolve, reject) => {
    const url = new URL(urlStr);
    const body = postData || '';
    const opts = {
      hostname: url.hostname,
      port: 443,
      path: url.pathname + url.search,
      method: method || 'GET',
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        'Accept': 'application/json, text/javascript, */*',
        'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8',
        'Referer': 'https://www.pilio.idv.tw/ltobig/list.asp',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Content-Length': Buffer.byteLength(body)
      }
    };

    const req = https.request(opts, (res) => {
      let data = '';
      res.setEncoding('utf8');
      res.on('data', chunk => { data += chunk; });
      res.on('end', () => resolve({ status: res.statusCode, body: data }));
    });
    req.on('error', reject);
    req.setTimeout(15000, () => { req.destroy(); reject(new Error('Request timeout')); });
    if (body) req.write(body);
    req.end();
  });
}

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

// Fetch a page of data using the JSON API
// Lindex: the index to fetch from (0 = oldest, use dex value from response to continue)
// Ldesc=0 means ascending (old to new), 1 means descending (new to old)
async function fetchJsonBatch(lindex, ldesc) {
  const postBody = `Lkind=ltobig&Lindex=${lindex}&Ldesc=${ldesc}`;
  const res = await httpsRequest(JSON_API, 'POST', postBody);

  if (res.status !== 200) {
    throw new Error(`HTTP ${res.status}`);
  }

  try {
    return JSON.parse(res.body);
  } catch (e) {
    // Try to clean and parse
    const cleaned = res.body.trim();
    if (!cleaned || cleaned === 'null' || cleaned === '{}') return null;
    throw new Error(`JSON parse error: ${e.message}, body: ${res.body.slice(0, 200)}`);
  }
}

// The paginated HTML endpoint as fallback
async function fetchPageHtml(pageNum) {
  const url = `https://www.pilio.idv.tw/ltobig/list.asp?indexpage=${pageNum}&orderby=old`;
  const res = await httpsRequest(url, 'GET');
  return res.body;
}

function parseLottoItem(item) {
  // item.date format: "01/01<br>07(一)" = MM/DD<br>YY(DayOfWeek)
  // item.num format: "02, 06, 11, 23, 38, 44" (6 numbers for 大樂透)
  // item.snum format: "07" (special number 特別號), or may be 7th number in item.num
  // item.dex: serial index number

  if (!item.date || !item.num) return null;

  // Parse date: split on <br> tag
  const dateParts = item.date.split(/<br\s*\/?>/i);
  const mmdd = dateParts[0] ? dateParts[0].replace(/<[^>]+>/g, '').trim() : '';
  const yearDow = dateParts[1] ? dateParts[1].replace(/<[^>]+>/g, '').trim() : '';

  // Extract MM and DD
  const mmddMatch = mmdd.match(/(\d{1,2})[\/-](\d{2})/);
  const mm = mmddMatch ? mmddMatch[1].padStart(2, '0') : '??';
  const dd = mmddMatch ? mmddMatch[2] : '??';

  // Extract 2-digit year and day-of-week: e.g. "07(一)"
  const ydMatch = yearDow.match(/(\d{1,2})\(([^)]+)\)/);
  let fullYear = '????';
  let dow = '';
  if (ydMatch) {
    const yy = parseInt(ydMatch[1]);
    fullYear = yy < 50 ? `20${String(yy).padStart(2, '0')}` : `19${String(yy).padStart(2,'0')}`;
    dow = ydMatch[2];
  }

  const dateStr = `${fullYear}/${mm}/${dd}${dow ? ' (' + dow + ')' : ''}`;

  // Clean and parse numbers (comma-separated: "02, 06, 11, 23, 38, 44")
  const numText = item.num.replace(/<[^>]+>/gi, '').replace(/&nbsp;/gi, ' ');
  const nums = numText.match(/\d{1,2}/g) || [];
  const validNums = nums
    .filter(n => parseInt(n) >= 1 && parseInt(n) <= 49)
    .map(n => String(parseInt(n)).padStart(2, '0'));

  // Determine special number (特別號):
  // Priority 1: item.snum field
  // Priority 2: 7th number in item.num
  let specialNum = null;

  // API returns special number in item.sp field
  if (item.sp) {
    const snumText = String(item.sp).replace(/<[^>]+>/gi, '').trim();
    const snumMatch = snumText.match(/\d{1,2}/);
    if (snumMatch) {
      const sn = parseInt(snumMatch[0]);
      if (sn >= 1 && sn <= 49) {
        specialNum = String(sn).padStart(2, '0');
      }
    }
  }

  // If no snum field and we have 7 numbers in item.num, use the 7th as special
  if (!specialNum && validNums.length >= 7) {
    specialNum = validNums[6];
  }

  // Need exactly 6 regular numbers
  const regularNums = validNums.slice(0, 6);

  if (regularNums.length < 6) return null;

  // If special not found yet, use a fallback placeholder
  if (!specialNum) {
    // Attempt to derive from raw data length — if exactly 6, special is unknown
    specialNum = '00';
  }

  return {
    date: dateStr,
    dateSort: `${fullYear}${mm}${dd}`, // for sorting
    numbers: regularNums,
    special: specialNum,
    dex: item.dex
  };
}

// ─── Try the JSON API approach ───────────────────────────────────────────────

async function testJsonApi() {
  console.log('Testing JSON API endpoint...');

  // Try with Lindex=0 (fetch from beginning)
  try {
    const result = await fetchJsonBatch(0, 0);
    if (result && result.lotto && result.lotto.length > 0) {
      console.log('JSON API works! Sample data:');
      console.log('Total in batch:', result.lotto.length);
      console.log('First item:', JSON.stringify(result.lotto[0]));
      console.log('Last item:', JSON.stringify(result.lotto[result.lotto.length - 1]));
      return { works: true, data: result };
    }
  } catch (e) {
    console.log('JSON API error:', e.message);
  }

  return { works: false };
}

async function main() {
  // First test the JSON API
  const test = await testJsonApi();

  if (test.works) {
    console.log('\nUsing JSON API to collect all data...');
    await collectViaJsonApi(test.data);
  } else {
    console.log('\nJSON API not available, falling back to HTML scraping...');
    await collectViaHtml();
  }
}

async function collectViaJsonApi(firstBatch) {
  const allDraws = [];

  // Process first batch
  for (const item of firstBatch.lotto) {
    const draw = parseLottoItem(item);
    if (draw) allDraws.push(draw);
  }

  let lastDex = firstBatch.lotto[firstBatch.lotto.length - 1].dex;
  let batchNum = 1;

  // Keep fetching batches until no more data
  while (true) {
    await sleep(300);
    batchNum++;
    process.stdout.write(`\r📥 Batch ${batchNum}, fetching after dex=${lastDex}... (${allDraws.length} draws)`);

    try {
      const batch = await fetchJsonBatch(lastDex, 0);

      if (!batch || !batch.lotto || batch.lotto.length === 0) {
        console.log('\n✅ No more data.');
        break;
      }

      let newCount = 0;
      for (const item of batch.lotto) {
        const draw = parseLottoItem(item);
        if (draw) { allDraws.push(draw); newCount++; }
      }

      if (newCount === 0) {
        console.log('\n✅ No new items in batch, done.');
        break;
      }

      lastDex = batch.lotto[batch.lotto.length - 1].dex;
    } catch (e) {
      console.log(`\n⚠️ Batch ${batchNum} failed: ${e.message}`);
      await sleep(1000);
    }
  }

  await finalize(allDraws);
}

async function collectViaHtml() {
  // Use the paginated HTML approach as fallback
  const page1 = await fetchPageHtml(1);
  const pageCountMatch = page1.match(/indexpage=(\d+)&orderby=old">\s*(?:最末頁|Last)/);
  const totalPages = pageCountMatch ? parseInt(pageCountMatch[1]) : 100;

  console.log(`📄 Total pages: ${totalPages}`);

  const allDraws = [];

  for (let p = 1; p <= totalPages; p++) {
    process.stdout.write(`\r📥 Page ${p}/${totalPages} (${allDraws.length} draws)`);
    try {
      const html = p === 1 ? page1 : await fetchPageHtml(p);
      // Try to extract embedded JSON
      const jsonMatch = html.match(/\{"lotto":\[[\s\S]*?\]\}/);
      if (jsonMatch) {
        try {
          const obj = JSON.parse(jsonMatch[0]);
          for (const item of obj.lotto) {
            const draw = parseLottoItem(item);
            if (draw) allDraws.push(draw);
          }
        } catch (e) {
          // ignore parse errors per page
        }
      }
      await sleep(250);
    } catch (e) {
      console.log(`\n⚠️ Page ${p} failed: ${e.message}`);
    }
  }

  await finalize(allDraws);
}

// ─── Prediction Model ────────────────────────────────────────────────────────

function computeFrequency(draws) {
  const freq = {};
  for (let i = 1; i <= 49; i++) freq[String(i).padStart(2, '0')] = 0;
  for (const draw of draws) {
    for (const n of draw.numbers) {
      const key = String(parseInt(n)).padStart(2, '0');
      if (freq[key] !== undefined) freq[key]++;
    }
  }
  return freq;
}

function computeSpecialFrequency(draws) {
  const freq = {};
  for (let i = 1; i <= 49; i++) freq[String(i).padStart(2, '0')] = 0;
  for (const draw of draws) {
    if (draw.special && draw.special !== '00') {
      const key = String(parseInt(draw.special)).padStart(2, '0');
      if (freq[key] !== undefined) freq[key]++;
    }
  }
  return freq;
}

function getLastSeenMap(draws) {
  const lastSeen = {};
  for (let i = 1; i <= 49; i++) lastSeen[String(i).padStart(2, '0')] = draws.length;
  for (let i = draws.length - 1; i >= 0; i--) {
    for (const num of draws[i].numbers) {
      const key = String(parseInt(num)).padStart(2, '0');
      if (lastSeen[key] === draws.length) lastSeen[key] = draws.length - 1 - i;
    }
  }
  return lastSeen;
}

function getSpecialLastSeenMap(draws) {
  const lastSeen = {};
  for (let i = 1; i <= 49; i++) lastSeen[String(i).padStart(2, '0')] = draws.length;
  for (let i = draws.length - 1; i >= 0; i--) {
    if (draws[i].special && draws[i].special !== '00') {
      const key = String(parseInt(draws[i].special)).padStart(2, '0');
      if (lastSeen[key] === draws.length) lastSeen[key] = draws.length - 1 - i;
    }
  }
  return lastSeen;
}

// 6 buckets spanning 1-49
const SPREAD_BUCKETS = [
  [1,2,3,4,5,6,7,8],
  [9,10,11,12,13,14,15,16],
  [17,18,19,20,21,22,23,24,25],
  [26,27,28,29,30,31,32,33],
  [34,35,36,37,38,39,40,41],
  [42,43,44,45,46,47,48,49],
];

function predictNextDraw(draws) {
  if (draws.length < 10) return { numbers: [], special: '00' };

  // M3 SpreadOverdue: for each of 6 buckets, pick the most-overdue regular number
  const lastSeen = getLastSeenMap(draws);

  const numbers = SPREAD_BUCKETS.map(bucket => {
    return bucket
      .map(n => ({ n: String(n).padStart(2, '0'), gap: lastSeen[String(n).padStart(2, '0')] || 0 }))
      .sort((a, b) => b.gap - a.gap)[0].n;
  }).sort((a, b) => parseInt(a) - parseInt(b));

  // Predict special number: most overdue from 1-49 that is NOT in the predicted regular numbers
  const specialLastSeen = getSpecialLastSeenMap(draws);
  const predictedSet = new Set(numbers);
  const specialCandidates = Object.entries(specialLastSeen)
    .filter(([k]) => !predictedSet.has(k))
    .sort((a, b) => b[1] - a[1]);
  const special = specialCandidates.length > 0 ? specialCandidates[0][0] : '00';

  return { numbers, special };
}

// ─── Markdown ─────────────────────────────────────────────────────────────────

function generateMarkdown(draws, prediction) {
  const freq = computeFrequency(draws);
  const lastSeen = getLastSeenMap(draws);
  const total = draws.length;

  const hot = Object.entries(freq).sort((a, b) => b[1] - a[1]).slice(0, 10);
  const cold = Object.entries(freq).sort((a, b) => a[1] - b[1]).slice(0, 10);
  const overdue = Object.entries(lastSeen).sort((a, b) => b[1] - a[1]).slice(0, 10);

  const lines = [
    '# 大樂透 歷史開獎號碼 & 預測分析',
    '',
    `> 資料來源：https://www.pilio.idv.tw/lotto/list.asp`,
    `> 共收錄 **${total}** 期開獎紀錄`,
    `> 產生時間：${new Date().toLocaleString('zh-TW')}`,
    '',
    '---',
    '',
    '## 🎯 下期預測號碼',
    '',
    '> 採用 M3 SpreadOverdue 模型：將 1–49 分成 6 區間，每區選最久未出現號碼',
    '>',
    '> ⚠️ 彩券具有不確定性，本預測僅供參考娛樂，請理性購買。',
    '',
    `### 預測號碼：**${prediction.numbers.join('  ·  ')}**  特別號：**${prediction.special}**`,
    '',
    '---',
    '',
    '## 📊 統計分析',
    '',
    '### 🔥 熱門號碼 Top 10',
    '',
    '| 號碼 | 出現次數 | 出現率 |',
    '|:---:|:---:|:---:|',
    ...hot.map(([num, count]) => `| **${num}** | ${count} | ${((count / total) * 100).toFixed(2)}% |`),
    '',
    '### 🧊 冷門號碼 Top 10',
    '',
    '| 號碼 | 出現次數 | 出現率 |',
    '|:---:|:---:|:---:|',
    ...cold.map(([num, count]) => `| **${num}** | ${count} | ${((count / total) * 100).toFixed(2)}% |`),
    '',
    '### ⏰ 最久未出現 (遲到號) Top 10',
    '',
    '| 號碼 | 距上次出現（期數）|',
    '|:---:|:---:|',
    ...overdue.map(([num, gap]) => `| **${num}** | ${gap === total ? '未曾出現' : gap} |`),
    '',
    '### 📈 全號碼出現頻率 (01–49)',
    '',
    '| 號碼 | 次數 | 出現率 | 號碼 | 次數 | 出現率 |',
    '|:---:|:---:|:---:|:---:|:---:|:---:|',
  ];

  const freqEntries = Object.entries(freq).sort((a, b) => parseInt(a[0]) - parseInt(b[0]));
  for (let i = 0; i < freqEntries.length; i += 2) {
    const [n1, c1] = freqEntries[i];
    const r1 = ((c1 / total) * 100).toFixed(2);
    if (i + 1 < freqEntries.length) {
      const [n2, c2] = freqEntries[i + 1];
      const r2 = ((c2 / total) * 100).toFixed(2);
      lines.push(`| ${n1} | ${c1} | ${r1}% | ${n2} | ${c2} | ${r2}% |`);
    } else {
      lines.push(`| ${n1} | ${c1} | ${r1}% | — | — | — |`);
    }
  }

  lines.push('', '---', '', '## 📋 歷史開獎紀錄（由新到舊）', '',
    '| # | 日期 | 號碼 1 | 號碼 2 | 號碼 3 | 號碼 4 | 號碼 5 | 號碼 6 | 特別號 |',
    '|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|');

  const reversed = [...draws].reverse();
  for (let i = 0; i < reversed.length; i++) {
    const draw = reversed[i];
    const idx = total - i;
    const [n1, n2, n3, n4, n5, n6] = draw.numbers;
    lines.push(`| ${idx} | ${draw.date} | ${n1} | ${n2} | ${n3} | ${n4} | ${n5} | ${n6} | **${draw.special}** |`);
  }

  return lines.join('\n');
}

async function finalize(allDraws) {
  console.log(`\n\n✅ Total draws collected: ${allDraws.length}`);

  // Dedup
  const seen = new Set();
  const unique = allDraws.filter(d => {
    const key = `${d.date}|${d.numbers.join(',')}`;
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  });

  console.log(`🔢 Unique draws: ${unique.length}`);

  // Validate: all draws should have exactly 6 unique numbers 1-49
  const valid = unique.filter(d => {
    if (d.numbers.length !== 6) return false;
    const nums = d.numbers.map(n => parseInt(n));
    const allValid = nums.every(n => n >= 1 && n <= 49);
    const allUnique = new Set(nums).size === 6;
    return allValid && allUnique;
  });

  console.log(`✔️  Valid draws (6 unique numbers 1-49): ${valid.length}`);

  if (valid.length < 50) {
    console.error('❌ Too few valid draws. Parsing may have failed.');
    console.log('Sample raw draws:');
    unique.slice(0, 5).forEach(d => console.log(d));
    process.exit(1);
  }

  console.log('\nSample draws (first 3):');
  valid.slice(0, 3).forEach(d => console.log(JSON.stringify(d)));
  console.log('Sample draws (last 3):');
  valid.slice(-3).forEach(d => console.log(JSON.stringify(d)));

  // Run prediction
  console.log('\n🔮 Running prediction model...');
  const prediction = predictNextDraw(valid);
  console.log(`🎯 Predicted next draw: ${prediction.numbers.join(', ')}  特別號: ${prediction.special}`);

  // Save markdown
  console.log('\n📝 Generating markdown...');
  const md = generateMarkdown(valid, prediction);
  fs.writeFileSync('lottery_649_history.md', md, 'utf8');
  console.log(`✅ Markdown saved: lottery_649_history.md (${(fs.statSync('lottery_649_history.md').size / 1024).toFixed(1)} KB)`);

  // Save JSON
  const jsonOut = {
    draws: valid,
    prediction,
    generatedAt: new Date().toISOString(),
    total: valid.length
  };
  fs.writeFileSync('lottery_649_data.json', JSON.stringify(jsonOut, null, 2), 'utf8');
  console.log(`✅ JSON saved: lottery_649_data.json`);
}

main().catch(console.error);