crawl-full.ts

import { chromium } from "playwright";
import * as path from "path";
import { chunkData } from "./lib/chunker";
import { extractPageData } from "./lib/extract";
import { extractStatistics } from "./lib/statistics";

interface CrawlResult {
  url: string;
  screenshot: string;
  title: string;
  description: string;
  favicon: string;
  viewport: string;
  headings: { level: number; text: string }[];
  images: string[];
  links: string[];
  buttons: string[];
  css: string[];
  statistics: { value: string; context: string }[];
  design: {
    colors: string[];
    fonts: string[];
    fontSizes: string[];
    spacing: string[];
    borderRadius: string[];
    animations: string[];
    gradients: string[];
  };
  text: string;
}

export async function crawlFull(url: string): Promise<CrawlResult> {
  const browser = await chromium.launch({
    headless: true,
    args: ["--no-sandbox", "--disable-dev-shm-usage"],
  });
  const context = await browser.newContext({ viewport: { width: 1440, height: 900 } });
  const page = await context.newPage();

  await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });
  await page.waitForTimeout(2000);

  const hostname = new URL(url).hostname.replace(/[^a-z0-9.-]/gi, "");
  const screenshotPath = path.join("/tmp", `crawl-${hostname}-${Date.now()}.png`);
  await page.screenshot({ path: screenshotPath, fullPage: true });

  const data = await extractPageData(page);
  const statistics = await extractStatistics(page);

  const vp = page.viewportSize();
  const viewport = vp ? `${vp.width}x${vp.height}` : "unknown";

  await browser.close();

  return { url, screenshot: screenshotPath, viewport, ...data, statistics };
}

if (require.main === module) {
  (async () => {
    const url = process.argv[2];
    if (!url) {
      console.error("Usage: npx tsx crawl-full.ts <url>");
      process.exit(1);
    }
    const result = await crawlFull(url);
    console.log(JSON.stringify(chunkData(result), null, 2));
  })();
}