Spaces:
Build error
Build error
File size: 1,811 Bytes
e27bcac | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | import { Also, AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from 'civkit';
import type { Request, Response } from 'express';
@Also({
openapi: {
operation: {
parameters: {
'X-Use-Sitemap': {
description: 'Use sitemap to crawl the website.',
in: 'header',
schema: { type: 'string' }
},
'X-Max-Depth': {
description: 'Max deep level to crawl.',
in: 'header',
schema: { type: 'string' }
},
'X-Max-Pages': {
description: 'Max number of pages to crawl.',
in: 'header',
schema: { type: 'string' }
},
}
}
}
})
export class AdaptiveCrawlerOptions extends AutoCastable {
@Prop({
default: true,
desc: 'Use sitemap to crawl the website.',
})
useSitemap!: boolean;
@Prop({
default: 10,
desc: 'Max number of pages to crawl.',
validate: (v: number) => v >= 1 && v <= 100,
})
maxPages!: number;
static override from(input: any) {
const instance = super.from(input) as AdaptiveCrawlerOptions;
const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {
req: Request,
res: Response,
} | undefined;
let maxPages = parseInt(ctx?.req.get('x-max-pages') || '');
if (!isNaN(maxPages) && maxPages > 0) {
instance.maxPages = maxPages <= 100 ? maxPages : 100;
}
const useSitemap = ctx?.req.get('x-use-sitemap');
if (useSitemap !== undefined) {
instance.useSitemap = Boolean(useSitemap);
}
return instance;
}
}
|