Shasha2020 commited on
Commit
c19ba65
·
1 Parent(s): 8f25fe1

update scrapping options

Browse files
backend/functions/src/dto/scrapping-options.ts CHANGED
@@ -396,7 +396,7 @@ export class CrawlerOptions extends AutoCastable {
396
  if (engine) {
397
  instance.engine = engine;
398
  }
399
- if (instance.noCache || !instance.isGeneralMarkdownRequest()) {
400
  instance.engine ??= ENGINE_TYPE.BROWSER;
401
  }
402
 
@@ -489,8 +489,8 @@ export class CrawlerOptions extends AutoCastable {
489
  return !CONTENT_FORMAT_VALUES.has(this.respondWith);
490
  }
491
 
492
- isGeneralMarkdownRequest() {
493
- if (this.respondWith !== CONTENT_FORMAT.CONTENT && this.respondWith !== CONTENT_FORMAT.MARKDOWN) {
494
  return false;
495
  }
496
  if (this.injectFrameScript?.length || this.injectPageScript?.length) {
 
396
  if (engine) {
397
  instance.engine = engine;
398
  }
399
+ if (instance.noCache || !instance.isTypicalRequest()) {
400
  instance.engine ??= ENGINE_TYPE.BROWSER;
401
  }
402
 
 
489
  return !CONTENT_FORMAT_VALUES.has(this.respondWith);
490
  }
491
 
492
+ isTypicalRequest() {
493
+ if (this.respondWith.includes(CONTENT_FORMAT.PAGESHOT) || this.respondWith.includes(CONTENT_FORMAT.SCREENSHOT)) {
494
  return false;
495
  }
496
  if (this.injectFrameScript?.length || this.injectPageScript?.length) {