lib/playwright-crawler.ts

import { chromium } from "playwright";

export async function crawlWithBrowser(url: string) {

  if (!url.startsWith("http")) {
    url = "https://" + url;
  }

  console.log("Browser crawling:", url);


  const browser = await chromium.launch({

    executablePath:
      "/usr/bin/chromium",

    headless: true,

    args: [
      "--no-sandbox",
      "--disable-setuid-sandbox",
      "--disable-dev-shm-usage",
      "--disable-gpu"
    ]

  });


  const page =
    await browser.newPage({
      viewport:{
        width:1440,
        height:900
      }
    });


  await page.goto(url, {
  waitUntil: "commit",
  timeout: 60000
})

  await page.waitForTimeout(3000)


  const data =
    await page.evaluate(()=>{


    const styles =
      [...document.styleSheets]
      .map(sheet=>{

        try{

          return [...sheet.cssRules]
          .map(r=>r.cssText)
          .join("\n");

        }catch{
          return "";
        }

      })
      .join("\n");



    const getStyles =
      [...document.querySelectorAll("*")]
      .map(el=>{

        const s =
          getComputedStyle(el);

        return {

          color:s.color,

          background:s.backgroundColor,

          font:s.fontFamily,

          radius:s.borderRadius,

          margin:s.margin,

          padding:s.padding

        };

      });



    return {

      title:
        document.title,


      html:
        document.documentElement.outerHTML
        .slice(0,10000),


      images:
        [...document.images]
        .map(i=>({
          src:i.src,
          alt:i.alt
        })),


      links:
        [...document.links]
        .map(a=>a.href),


      headings:
        [...document.querySelectorAll(
          "h1,h2,h3"
        )]
        .map(x=>x.textContent?.trim()),


      buttons:
        [...document.querySelectorAll(
          "button,a"
        )]
        .map(x=>x.textContent?.trim())
        .filter(Boolean),


      css:styles,


      computedStyles:getStyles,


      text:
        document.body.innerText
        .replace(/\s+/g," ")
        .slice(0,5000)

    };


  });


  await browser.close();


  return {
    url,
    ...data
  };

}