Update index.js

index.js

@@ -1,201 +1,77 @@
-const express = require("express")
+const express = require("express");
 const app = express();
-const puppeteer = require("puppeteer");
+const puppeteer = require("puppeteer-extra");
+const StealthPlugin = require("puppeteer-extra-plugin-stealth");
 const bodyParser = require("body-parser");
 const mutler = require("multer");
 
-// parse application/json
 app.use(bodyParser.json());
-
-// parse application/x-www-form-urlencoded
 app.use(bodyParser.urlencoded({ extended: false }));
-
-// app.use(express.urlencoded());
 app.use(mutler().array(""));
 
+puppeteer.use(StealthPlugin());
 
 const initBrowser = puppeteer.launch({
-    executablePath: process.env.CHROME_BIN || null,
-    defaultViewport: null,
-    headless: true,
+  executablePath: process.env.CHROME_BIN || null,
+  defaultViewport: null,
+  headless: true,
 });
 
-async function addRequestFilter(page) {
-
-    const resourcetypes = ['document'];
-
-    await page.setRequestInterception(true);
-    // page.
-    page.on("request", (request) => {
-        // return request.continue();
-        if (resourcetypes.includes(request.resourceType())) {
-            request.continue();
-        }
-        else {
-            request.abort();
-        }
-    });
-
-    return page;
-}
-
-const getData = async (url) => {
-
-    // Open a new page
-    const browser = await initBrowser;
-    var page = await browser.newPage();
-    page = await addRequestFilter(page);
-
-    await page.goto(url, {
-        // waitUntil: "domcontentloaded",
-    });
-    // console.log(await page.content());
-
-    const title = await page.evaluate(() => {
-        const elem = document.querySelector("span.B_NuCI");
-        if (elem)
-            return elem.textContent;
-    });
-    const price = await page.evaluate(() => {
-        const elem = document.querySelector("div._30jeq3._16Jk6d");
-        if (elem)
-            return elem.textContent;
-    });
-
-
-    const image = await page.evaluate(() => {
-        const elem = document.querySelector("div.CXW8mj._3nMexc>img");
-        if (elem)
-            return elem.src;
-    });
-
-
-    page.close();
-    return {
-        title: title,
-        price: price,
-        image: image,
-    };
-};
-
-const getHTML = async (url, headers = null) => {
-
-    const browser = await initBrowser;
-    var page = await browser.newPage();
-    page = await addRequestFilter(page);
-    if (headers) {
-        // console.log("headers",headers);
-        await page.setExtraHTTPHeaders(headers);
-    }
-
-    await page.goto(url, {
-        // waitUntil: "domcontentloaded",
-    });
-
-    return await page.content();
-
-}
-const getScreenshot = async (url, headers = null) => {
-
-    const browser = await initBrowser;
-    var page = await browser.newPage();
-
-    if (headers) {
-        // console.log("headers",headers);
-        await page.setExtraHTTPHeaders(headers);
-    }
-    
-    await page.goto(url, {
-        waitUntil: "domcontentloaded",
-    });
-
-    const image = await page.screenshot({
-        type: "png",
-    });
-
-    const html = await page.content();
-
-    page.close();
-    return { image, html };
-
+async function getDataWebsite(url) {
+  const browser = await initBrowser;
+  const page = await browser.newPage();
+  await page.setViewport({width: 1920, height: 1080});
+  
+  const response = await page.goto(url, {
+    timeout: 0,
+    waitUntil: 'networkidle0',
+  });
+  
+  const headers = response.headers();
+  const cookies = await page.cookies();
+  const cookieString = cookies?.map(cookie => `${cookie.name}=${cookie.value}`).join('; ');
+  const content_html = await page.evaluate(() => {
+    return new XMLSerializer().serializeToString(document);
+  });
+  const screenData = await page.screenshot({encoding: 'binary', type: 'jpeg', quality: 100});
+  await page.close();
+  await browser.close();
+  return {
+    headers,
+    cookies,
+    cookieString,
+    content_html,
+    screenData,
+  }
 }
 
 app.get("/", async (req, res) => {
-    res.send('go to /test');
-})
-
-url = "https://www.flipkart.com/apple-iphone-15-blue-128-gb/p/itmbf14ef54f645d";
-
-
-app.get("/test", async (req, res) => {
-    const data = await getData(url);
-    res.type("json");
-    res.send(JSON.stringify(data));
-})
-
-app.get("/testhtml", async (req, res) => {
-    const html = await getHTML(url);
-    res.type("json");
-    res.send(JSON.stringify({
-        html: html
-    }));
-})
-app.get("/testscreenshot", async (req, res) => {
-    const image = await getScreenshot(url);
-    // convert buffer to base64 string
-    const base64Image = await image.toString('base64');
-
-    res.type("json");
-    return res.send(JSON.stringify({
-        "screenshot": "data:image/png;base64," + base64Image
-    }));
-})
-app.post("/html", async (req, res) => {
-    const data = req.body;
-    if (!("url" in data)) {
-        res.type("json");
-        return res.send(JSON.stringify({
-            "error": "no url parameter in request",
-        }));
-    }
-    const { url, headers } = data;
-
-    try {
-        const html = await getHTML(url, headers);
-        res.type("json").send(JSON.stringify({
-            html: html
-        }));
-    }
-    catch (e) {
-        return res.type("json").send(JSON.stringify({
-            "error": "can't open page",
-        }));
-    }
-})
-app.post("/screenshot", async (req, res) => {
-    const data = req.body;
-    if (!("url" in data)) {
-        return res.type("json").send(JSON.stringify({
-            "error": "no url parameter in request",
-        }));
-    }
-    const url = data['url'];
-
-    try {
-        const { image, html } = await getScreenshot(url);
-        // convert buffer to base64 string
-        const base64Image = await image.toString('base64');
+  res.send("Hello World");
+});
 
-        return res.type("json").send(JSON.stringify({
-            "base64": "data:image/png;base64," + base64Image,
-            "html":html,
-        }));
-    }
-    catch (e) {
-        return res.type("json").send(JSON.stringify({
-            "error": "can't open page",
-        }));
-    }
-})
+app.post("/getDataWebsite", async (req, res) => {
+  const data = req.body;
+  if (!("url" in data)) {
+    return res.type("json").send(
+      JSON.stringify({
+        error: "no url parameter in request",
+      })
+    );
+  }
+  const url = data["url"];
+
+  try {
+    const response = await getDataWebsite(url);
+    return res.type("json").send(
+      JSON.stringify(response)
+    );
+  } catch (e) {
+    return res.type("json").send(
+      JSON.stringify({
+        error: "can't open page",
+      })
+    );
+  }
+});
 
 app.listen(8080, () => console.log("Server running at port 8080"));