src/scraper.js

// src/scraper.js
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
const { Readability } = require("@mozilla/readability");
const { JSDOM } = require("jsdom");
const TurndownService = require("turndown");
const { gfm } = require("turndown-plugin-gfm");

puppeteer.use(StealthPlugin());

// ── Turndown ──────────────────────────────────────────────────────────────
const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
td.use(gfm);
td.addRule("removeNoise", {
  filter: ["script", "style", "noscript", "svg", "iframe"],
  replacement: () => "",
});

// ── HTML → Clean Markdown (Readability দিয়ে) ─────────────────────────────
async function pageToMarkdown(page) {
  const html = await page.content();
  const url = page.url();
  try {
    const dom = new JSDOM(html, { url });
    const reader = new Readability(dom.window.document);
    const article = reader.parse();
    if (article?.content) return td.turndown(article.content).replace(/\n{3,}/g, "\n\n").trim();
  } catch (_) {}
  // Readability fail করলে fallback
  const bodyHtml = await page.evaluate(() => document.body?.innerHTML || "");
  return td.turndown(bodyHtml).replace(/\n{3,}/g, "\n\n").trim();
}

// ── Page metadata ─────────────────────────────────────────────────────────
async function getPageMeta(page) {
  return page.evaluate(() => {
    const getMeta = (n) =>
      document.querySelector(`meta[name="${n}"]`)?.content ||
      document.querySelector(`meta[property="og:${n}"]`)?.content || null;

    const links = Array.from(document.querySelectorAll("a[href]"))
      .map((a) => ({ text: a.innerText.trim().slice(0, 100), href: a.href }))
      .filter((l) => l.text && l.href.startsWith("http"))
      .slice(0, 50);

    const inputs = Array.from(document.querySelectorAll("input,textarea,select")).map((el) => ({
      tag: el.tagName.toLowerCase(),
      type: el.type || null,
      name: el.name || el.id || null,
      placeholder: el.placeholder || null,
      selector: el.id ? `#${el.id}` : el.name ? `[name="${el.name}"]` : el.tagName.toLowerCase(),
    }));

    const buttons = Array.from(document.querySelectorAll("button,[role=button],input[type=submit]"))
      .map((b) => ({
        text: (b.innerText?.trim() || b.value || "").slice(0, 60),
        selector: b.id ? `#${b.id}` : b.className ? `.${b.className.split(" ")[0]}` : "button",
      }))
      .filter((b) => b.text).slice(0, 20);

    const headings = Array.from(document.querySelectorAll("h1,h2,h3"))
      .map((h) => ({ level: h.tagName, text: h.innerText.trim() }))
      .filter((h) => h.text).slice(0, 20);

    return {
      title: document.title,
      url: window.location.href,
      description: getMeta("description"),
      keywords: getMeta("keywords"),
      links, inputs, buttons, headings,
    };
  });
}

// ── Browser launch (stealth) ──────────────────────────────────────────────
async function launchBrowser(proxy = null) {
  const args = [
    "--no-sandbox", "--disable-setuid-sandbox",
    "--disable-dev-shm-usage", "--disable-gpu",
    "--window-size=1280,800",
    "--disable-blink-features=AutomationControlled",
  ];
  if (proxy) args.push(`--proxy-server=${proxy}`);

  return puppeteer.launch({
    executablePath: process.env.CHROMIUM_PATH || "/usr/bin/chromium",
    headless: "new",
    args,
  });
}

// ── Stealth page setup ────────────────────────────────────────────────────
async function newStealthPage(browser) {
  const page = await browser.newPage();
  await page.setViewport({ width: 1280, height: 800 });
  await page.setUserAgent(
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
  );
  await page.evaluateOnNewDocument(() => {
    Object.defineProperty(navigator, "webdriver", { get: () => undefined });
    Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3] });
    window.chrome = { runtime: {} };
  });
  return page;
}

// ── Smart wait — SPA support ──────────────────────────────────────────────
async function smartWait(page, options = {}) {
  const { waitFor, waitForSelector, networkIdle = false } = options;
  if (networkIdle) {
    await page.waitForLoadState?.("networkidle").catch(() =>
      new Promise((r) => setTimeout(r, 2000))
    );
  }
  if (waitForSelector) await page.waitForSelector(waitForSelector, { timeout: 10000 }).catch(() => {});
  if (waitFor > 0) await new Promise((r) => setTimeout(r, Math.min(waitFor, 15000)));
}

module.exports = { launchBrowser, newStealthPage, pageToMarkdown, getPageMeta, smartWait, td };