tweak: try alleviate oom killed issue

backend/functions/src/cloud-functions/crawler.ts

@@ -130,11 +130,11 @@ export class CrawlerHost extends RPCHost {
     })
     @CloudHTTPv2({
         runtime: {
-            memory: '8GiB',
+            memory: '4GiB',
             cpu: 4,
             timeoutSeconds: 300,
-            concurrency: 22,
-            maxInstances: 455,
+            concurrency: 8,
+            maxInstances: 1250,
             minInstances: 1,
         },
         tags: ['Crawler'],

backend/functions/src/services/puppeteer.ts

@@ -9,7 +9,7 @@ import puppeteer from 'puppeteer-extra';
 
 import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
 import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
-import { SecurityCompromiseError, ServiceCrashedError } from '../shared/lib/errors';
+import { SecurityCompromiseError, ServiceCrashedError, ServiceNodeResourceDrainError } from '../shared/lib/errors';
 import { TimeoutError } from 'puppeteer';
 import _ from 'lodash';
 const tldExtract = require('tld-extract');
@@ -285,7 +285,13 @@ export class PuppeteerControl extends AsyncService {
         await this.serviceReady();
         const dedicatedContext = await this.browser.createBrowserContext();
         const sn = this._sn++;
-        const page = await dedicatedContext.newPage();
+        let page
+        try {
+            page = await dedicatedContext.newPage();
+        } catch (err: any) {
+            this.logger.warn(`Failed to create page ${sn}`, { err: marshalErrorLike(err) });
+            throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`);
+        }
         const preparations = [];
 
         // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));

thinapps-shared

@@ -1 +1 @@
-Subproject commit d2b0fbf184b4c77e80e8d1dd36b3f4d1807e0e09
+Subproject commit 2ce99a95bff6ea07f48e3388e270b2126d752d0c