lib/crawler.ts

import { Browser } from "playwright";
import * as path from "path";
import { extractPageData } from "./extract";
import { extractStatistics } from "./statistics";

export async function crawlPage(browser: Browser, url: string) {
  const context = await browser.newContext({ viewport: { width: 1440, height: 900 } });

  try {
    const page = await context.newPage();

    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });
    await page.waitForTimeout(2000);

    const hostname = new URL(url).hostname.replace(/[^a-z0-9.-]/gi, "");
    const screenshotPath = path.join("/tmp", `crawl-${hostname}-${Date.now()}.png`);
    await page.screenshot({ path: screenshotPath, fullPage: true });

    const data = await extractPageData(page);
    const statistics = await extractStatistics(page);

    const vp = page.viewportSize();
    const viewport = vp ? `${vp.width}x${vp.height}` : "unknown";

    return { url, screenshot: screenshotPath, viewport, ...data, statistics };
  } finally {
    await context.close(); // always runs — success or failure
  }
}