Spaces:
Build error
Build error
chore: fix abuse timing
Browse files
backend/functions/src/cloud-functions/crawler.ts
CHANGED
|
@@ -562,6 +562,10 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
|
|
| 562 |
}
|
| 563 |
|
| 564 |
if (!uid) {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 565 |
const blockade = (await DomainBlockade.fromFirestoreQuery(
|
| 566 |
DomainBlockade.COLLECTION
|
| 567 |
.where('domain', '==', urlToCrawl.hostname.toLowerCase())
|
|
@@ -571,10 +575,6 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
|
|
| 571 |
if (blockade) {
|
| 572 |
throw new SecurityCompromiseError(`Domain ${urlToCrawl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`);
|
| 573 |
}
|
| 574 |
-
if (urlToCrawl.protocol === 'http:' && (!urlToCrawl.pathname || urlToCrawl.pathname === '/') &&
|
| 575 |
-
crawlerOptions.respondWith !== 'default') {
|
| 576 |
-
throw new SecurityCompromiseError(`Your request is categorized as abuse. Please don't abuse our service. If you are sure you are not abusing, please authenticate yourself with an API key.`);
|
| 577 |
-
}
|
| 578 |
|
| 579 |
}
|
| 580 |
const crawlOpts = this.configure(crawlerOptions);
|
|
|
|
| 562 |
}
|
| 563 |
|
| 564 |
if (!uid) {
|
| 565 |
+
if (urlToCrawl.protocol === 'http:' && (!urlToCrawl.pathname || urlToCrawl.pathname === '/') &&
|
| 566 |
+
crawlerOptions.respondWith !== 'default') {
|
| 567 |
+
throw new SecurityCompromiseError(`Your request is categorized as abuse. Please don't abuse our service. If you are sure you are not abusing, please authenticate yourself with an API key.`);
|
| 568 |
+
}
|
| 569 |
const blockade = (await DomainBlockade.fromFirestoreQuery(
|
| 570 |
DomainBlockade.COLLECTION
|
| 571 |
.where('domain', '==', urlToCrawl.hostname.toLowerCase())
|
|
|
|
| 575 |
if (blockade) {
|
| 576 |
throw new SecurityCompromiseError(`Domain ${urlToCrawl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`);
|
| 577 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 578 |
|
| 579 |
}
|
| 580 |
const crawlOpts = this.configure(crawlerOptions);
|