lib/extract.ts

import { Page } from "playwright";

export async function extractPageData(page: Page) {
  return await page.evaluate(() => {
    const title = document.title;
    const description =
      document.querySelector('meta[name="description"]')?.getAttribute("content") ||
      document.querySelector('meta[property="og:description"]')?.getAttribute("content") ||
      "";

    const faviconEl =
      document.querySelector('link[rel="icon"]') ||
      document.querySelector('link[rel="shortcut icon"]') ||
      document.querySelector('link[rel="apple-touch-icon"]');
    const favicon = faviconEl
      ? new URL(faviconEl.getAttribute("href") || "", location.href).href
      : new URL("/favicon.ico", location.href).href;

    const headings: { level: number; text: string }[] = [];
    const seenHeadings = new Set<string>();
    document.querySelectorAll("h1, h2, h3, h4, h5, h6").forEach((el) => {
      const t = el.textContent?.trim();
      const key = `${el.tagName}-${t}`;
      if (t && !seenHeadings.has(key)) {
        seenHeadings.add(key);
        headings.push({ level: Number(el.tagName[1]), text: t });
      }
    });

    const images = [...new Set(
      Array.from(document.querySelectorAll("img[src]")).map((img) =>
        new URL(img.getAttribute("src") || "", location.href).href
      )
    )].slice(0, 100);

    const links = [...new Set(
      Array.from(document.querySelectorAll("a[href]")).map((a) =>
        new URL(a.getAttribute("href") || "", location.href).href
      )
    )].slice(0, 150);

    const buttons = [...new Set(
      Array.from(document.querySelectorAll(
        "button, input[type='submit'], input[type='button'], a[class*='btn'], a[role='button']"
      )).map((el) => (el.textContent || (el as HTMLInputElement).value || "").trim())
        .filter((t) => t.length > 0 && t.length <= 60)
    )].slice(0, 50);

    const css = [...new Set(
      Array.from(document.querySelectorAll('link[rel="stylesheet"]')).map((l) =>
        new URL(l.getAttribute("href") || "", location.href).href
      )
    )];

    document.querySelectorAll("nav, header, footer, aside, script, style, noscript").forEach((el) => el.remove());

    const colors = new Set<string>();
    const fonts = new Set<string>();
    const fontSizes = new Set<string>();
    const spacing = new Set<string>();
    const borderRadius = new Set<string>();
    const animations = new Set<string>();
    const gradients = new Set<string>();

    const elements = Array.from(document.querySelectorAll("*")).slice(0, 800);
    for (const el of elements) {
      const cs = getComputedStyle(el);

      [cs.color, cs.backgroundColor, cs.borderTopColor, cs.borderRightColor, cs.borderBottomColor, cs.borderLeftColor]
        .forEach((c) => {
          if (c && c !== "rgba(0, 0, 0, 0)" && c !== "transparent") colors.add(c);
        });

      if (cs.fontFamily) fonts.add(cs.fontFamily.split(",")[0].replace(/['"]/g, "").trim());
      if (cs.fontSize) fontSizes.add(cs.fontSize);

      [cs.marginTop, cs.marginBottom, cs.marginLeft, cs.marginRight,
       cs.paddingTop, cs.paddingBottom, cs.paddingLeft, cs.paddingRight].forEach((v) => {
        if (v && v !== "0px") spacing.add(v);
      });

      if (cs.borderRadius && cs.borderRadius !== "0px") borderRadius.add(cs.borderRadius);

      if (cs.animationName && cs.animationName !== "none") {
        animations.add(`${cs.animationName} ${cs.animationDuration} ${cs.animationTimingFunction}`);
      }
      if (cs.transitionProperty && cs.transitionProperty !== "all" && cs.transitionDuration !== "0s") {
        animations.add(`transition: ${cs.transitionProperty} ${cs.transitionDuration}`);
      }

      if (cs.backgroundImage && cs.backgroundImage.includes("gradient")) {
        gradients.add(cs.backgroundImage);
      }
    }

    const text = (document.body.innerText || "")
      .replace(/\n{3,}/g, "\n\n")
      .replace(/[ \t]{2,}/g, " ")
      .trim();

    return {
      title, description, favicon, headings, images, links, buttons, css,
      design: {
        colors: Array.from(colors).slice(0, 30),
        fonts: Array.from(fonts).slice(0, 15),
        fontSizes: Array.from(fontSizes).sort((a, b) => parseFloat(a) - parseFloat(b)),
        spacing: Array.from(spacing).slice(0, 30),
        borderRadius: Array.from(borderRadius).slice(0, 15),
        animations: Array.from(animations).slice(0, 30),
        gradients: Array.from(gradients).slice(0, 15),
      },
      text: text.slice(0, 8000),
    };
  });
}