fix

backend/functions/.puppeteerrc.cjs

@@ -4,7 +4,7 @@ let config = {};
 if (!process.env.FUNCTIONS_EMULATOR) {
     config = {
         // Changes the cache location for Puppeteer.
-        cacheDirectory: join(__dirname, '.cache', 'puppeteer'),
+        cacheDirectory: join(__dirname, 'puppeteer'),
     };
 }
 

backend/functions/src/cloud-functions/crawler.ts

@@ -65,9 +65,9 @@ ${this.content}
     })
     @CloudHTTPv2({
         runtime: {
-            memory: '4GiB',
+            memory: '8GiB',
             timeoutSeconds: 540,
-            concurrency: 4,
+            concurrency: 16,
         },
         httpMethod: ['get', 'post'],
         returnType: [String, OutputServerEventStream],

backend/functions/src/services/puppeteer.ts

@@ -49,11 +49,12 @@ export class PuppeteerControl extends AsyncService {
             return page.browser().connected && !page.isClosed();
         }
     }, {
-        max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 4),
+        max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 16),
         min: 1,
         acquireTimeoutMillis: 60_000,
         testOnBorrow: true,
         testOnReturn: true,
+        autostart: false,
     });
 
     constructor(protected globalLogger: Logger) {
@@ -63,6 +64,8 @@ export class PuppeteerControl extends AsyncService {
     override async init() {
         await this.dependencyReady();
 
+        this.pagePool.start();
+
         if (this.browser) {
             if (this.browser.connected) {
                 await this.browser.close();
@@ -72,7 +75,13 @@ export class PuppeteerControl extends AsyncService {
         }
         this.browser = await puppeteer.launch({
             headless: true,
-            timeout: 45_000
+            timeout: 10_000
+        }).catch((err) => {
+            this.logger.error(`Unknown firebase issue, just die fast, quitting process.`, { err });
+            process.nextTick(()=> {
+                process.exit(1);
+            });
+            return Promise.reject(err);
         });
         this.browser.once('disconnected', () => {
             this.logger.warn(`Browser disconnected`);
@@ -90,7 +99,7 @@ export class PuppeteerControl extends AsyncService {
         const page = await dedicatedContext.newPage();
         const preparations = [];
 
-        preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));
+        // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));
         preparations.push(page.setBypassCSP(true));
         preparations.push(page.setViewport({ width: 1920, height: 1080 }));
         preparations.push(page.exposeFunction('reportSnapshot', (snapshot: any) => {