scripts/fetch_sp500.ts

import * as cheerio from 'cheerio';
import fs from 'fs';
import path from 'path';

async function getSP500FromWiki() {
  const url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies';
  console.log('Fetching S&P 500 from:', url);

  try {
    const res = await fetch(url);
    if (!res.ok) throw new Error(`Status: ${res.status}`);

    const html = await res.text();
    const $ = cheerio.load(html);
    const symbols: string[] = [];

    // The first table usually has id="constituents"
    const table = $('#constituents');

    if (table.length === 0) {
      throw new Error('Could not find constituents table on Wikipedia page.');
    }

    // Iterate over rows in the tbody
    table.find('tbody tr').each((i, row) => {
      // Skip header row if it's th
      const firstCell = $(row).find('td').first();
      let symbol = firstCell.text().trim();

      // Yahoo Finance prefers dashes for classes like BRK.B -> BRK-B
      if (symbol) {
        symbol = symbol.replace(/\./g, '-');
        symbols.push(symbol);
      }
    });

    console.log(`Successfully extracted ${symbols.length} symbols.`);
    // Sample
    console.log(symbols.slice(0, 5));

    fs.writeFileSync(path.resolve(__dirname, 'sp500_symbols.json'), JSON.stringify(symbols, null, 2));
    console.log('Saved to scripts/sp500_symbols.json');

  } catch (e: any) {
    console.error('Scrape failed:', e.message);
  }
}

getSP500FromWiki();