lib/browser-crawler.ts

import { chromium } from "playwright";
import fs from "fs";

export async function crawlBrowser(url:string){

console.log("AI Browser crawling:",url);


const browserPath =
"/root/.cache/puppeteer/chrome/linux_arm-148.0.7778.97/chrome-linux64/chrome";


const browser =
await chromium.launch({
  executablePath: browserPath,
  headless:true,
  args:[
    "--no-sandbox",
    "--disable-setuid-sandbox",
    "--disable-dev-shm-usage"
  ]
});


const page = await browser.newPage({
 viewport:{
   width:1440,
   height:900
 }
});


await page.goto(url,{
 waitUntil:"commit",
 timeout:60000
});


const data = await page.evaluate(()=>{


const pick=(arr:string[]) =>
Array.from(new Set(arr.filter(Boolean)));


const elements =
Array.from(document.querySelectorAll("*"));


const styles =
elements.map(e=>getComputedStyle(e));


return {


title:
document.title,


description:
document.querySelector(
'meta[name="description"]'
)?.getAttribute("content") || "",


viewport:
document.querySelector(
'meta[name="viewport"]'
)?.getAttribute("content") || "",


favicon:
(document.querySelector(
'link[rel="icon"]'
) as HTMLLinkElement)?.href || "",



headings:
Array.from(
document.querySelectorAll("h1,h2,h3,h4")
)
.map(x=>x.textContent?.trim())
.filter(Boolean),



images:
Array.from(
document.images
)
.map(x=>({
 src:x.src,
 alt:x.alt,
 width:x.width,
 height:x.height
})),


links:
Array.from(
document.querySelectorAll("a")
)
.map(x=>(x as HTMLAnchorElement).href)
.filter(Boolean),



buttons:
Array.from(
document.querySelectorAll(
"button,a"
)
)
.map(x=>x.textContent?.trim())
.filter(Boolean),



css:
Array.from(
document.styleSheets
)
.map((x:any)=>x.href)
.filter(Boolean),



design:{


colors:
pick(
styles.map(
s=>s.color
)
.concat(
styles.map(
s=>s.backgroundColor
)
)
),



fonts:
pick(
styles.map(
s=>s.fontFamily
)
),



fontSizes:
pick(
styles.map(
s=>s.fontSize
)
),



spacing:
pick(
styles.map(
s=>`${s.margin}|${s.padding}`
)
),



borderRadius:
pick(
styles.map(
s=>s.borderRadius
)
),



animations:
pick(
styles.map(
s=>s.animation
)
),



gradients:
pick(
styles.map(
s=>s.backgroundImage
)
.filter(x=>x.includes("gradient"))
)

},


text:
document.body.innerText
.replace(/\s+/g," ")
.slice(0,5000)


};


});



const screenshot =
`/tmp/crawl-${Date.now()}.png`;


await page.screenshot({
 path:screenshot,
 fullPage:true
});



await browser.close();


return {
 url,
 screenshot,
 ...data
};


}