"use strict"; var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) { var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d; if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc); else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r; return c > 3 && r && Object.defineProperty(target, key, r), r; }; var __metadata = (this && this.__metadata) || function (k, v) { if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v); }; var _a; Object.defineProperty(exports, "__esModule", { value: true }); exports.CrawlerOptionsHeaderOnly = exports.CrawlerOptions = exports.BASE_URL_MODES = exports.IMAGE_RETENTION_MODES = exports.RESPOND_TIMING = exports.ENGINE_TYPE = exports.CONTENT_FORMAT = void 0; const civ_rpc_1 = require("civkit/civ-rpc"); const fancy_file_1 = require("civkit/fancy-file"); const set_cookie_parser_1 = require("set-cookie-parser"); const turndown_tweakable_options_1 = require("./turndown-tweakable-options"); var CONTENT_FORMAT; (function (CONTENT_FORMAT) { CONTENT_FORMAT["CONTENT"] = "content"; CONTENT_FORMAT["MARKDOWN"] = "markdown"; CONTENT_FORMAT["HTML"] = "html"; CONTENT_FORMAT["TEXT"] = "text"; CONTENT_FORMAT["PAGESHOT"] = "pageshot"; CONTENT_FORMAT["SCREENSHOT"] = "screenshot"; CONTENT_FORMAT["VLM"] = "vlm"; CONTENT_FORMAT["READER_LM"] = "readerlm-v2"; })(CONTENT_FORMAT || (exports.CONTENT_FORMAT = CONTENT_FORMAT = {})); var ENGINE_TYPE; (function (ENGINE_TYPE) { ENGINE_TYPE["AUTO"] = "auto"; ENGINE_TYPE["BROWSER"] = "browser"; ENGINE_TYPE["CURL"] = "curl"; ENGINE_TYPE["CF_BROWSER_RENDERING"] = "cf-browser-rendering"; })(ENGINE_TYPE || (exports.ENGINE_TYPE = ENGINE_TYPE = {})); var RESPOND_TIMING; (function (RESPOND_TIMING) { RESPOND_TIMING["HTML"] = "html"; RESPOND_TIMING["VISIBLE_CONTENT"] = "visible-content"; RESPOND_TIMING["MUTATION_IDLE"] = "mutation-idle"; RESPOND_TIMING["RESOURCE_IDLE"] = "resource-idle"; RESPOND_TIMING["MEDIA_IDLE"] = "media-idle"; RESPOND_TIMING["NETWORK_IDLE"] = "network-idle"; })(RESPOND_TIMING || (exports.RESPOND_TIMING = RESPOND_TIMING = {})); const CONTENT_FORMAT_VALUES = new Set(Object.values(CONTENT_FORMAT)); exports.IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p']; const IMAGE_RETENTION_MODE_VALUES = new Set(exports.IMAGE_RETENTION_MODES); exports.BASE_URL_MODES = ['initial', 'final']; const BASE_URL_MODE_VALUES = new Set(exports.BASE_URL_MODES); class Viewport extends civ_rpc_1.AutoCastable { } __decorate([ (0, civ_rpc_1.Prop)({ default: 1024 }), __metadata("design:type", Number) ], Viewport.prototype, "width", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: 1024 }), __metadata("design:type", Number) ], Viewport.prototype, "height", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Number) ], Viewport.prototype, "deviceScaleFactor", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Boolean) ], Viewport.prototype, "isMobile", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Boolean) ], Viewport.prototype, "isLandscape", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Boolean) ], Viewport.prototype, "hasTouch", void 0); let CrawlerOptions = class CrawlerOptions extends civ_rpc_1.AutoCastable { static from(input) { const instance = super.from(input); const ctx = Reflect.get(input, civ_rpc_1.RPC_CALL_ENVIRONMENT); const customMode = ctx?.get('x-respond-with') || ctx?.get('x-return-format'); if (customMode) { instance.respondWith = customMode; } if (instance.respondWith) { instance.respondWith = instance.respondWith.toLowerCase(); } if (instance.respondWith?.includes('lm')) { if (instance.respondWith.includes('content') || instance.respondWith.includes('markdown')) { throw new civ_rpc_1.ParamValidationError({ path: 'respondWith', message: `LM formats conflicts with content/markdown.`, }); } } const locale = ctx?.get('x-locale'); if (locale) { instance.locale = locale; } const referer = ctx?.get('x-referer'); if (referer) { instance.referer = referer; } const withGeneratedAlt = ctx?.get('x-with-generated-alt'); if (withGeneratedAlt) { instance.withGeneratedAlt = Boolean(withGeneratedAlt); } const withLinksSummary = ctx?.get('x-with-links-summary'); if (withLinksSummary) { if (withLinksSummary === 'all') { instance.withLinksSummary = withLinksSummary; } else { instance.withLinksSummary = Boolean(withLinksSummary); } } const withImagesSummary = ctx?.get('x-with-images-summary'); if (withImagesSummary) { instance.withImagesSummary = Boolean(withImagesSummary); } const retainImages = ctx?.get('x-retain-images'); if (retainImages && IMAGE_RETENTION_MODE_VALUES.has(retainImages)) { instance.retainImages = retainImages; } if (instance.withGeneratedAlt) { instance.retainImages = 'all_p'; } const noCache = ctx?.get('x-no-cache'); if (noCache) { instance.noCache = Boolean(noCache); } if (instance.noCache && instance.cacheTolerance === undefined) { instance.cacheTolerance = 0; } let cacheTolerance = parseInt(ctx?.get('x-cache-tolerance') || ''); if (!isNaN(cacheTolerance)) { instance.cacheTolerance = cacheTolerance; } const noGfm = ctx?.get('x-no-gfm'); if (noGfm) { instance.noGfm = noGfm === 'table' ? noGfm : Boolean(noGfm); } let timeoutSeconds = parseInt(ctx?.get('x-timeout') || ''); if (!isNaN(timeoutSeconds) && timeoutSeconds > 0) { instance.timeout = timeoutSeconds <= 180 ? timeoutSeconds : 180; } else if (ctx?.get('x-timeout')) { instance.timeout = null; } const removeSelector = ctx?.get('x-remove-selector')?.split(', ').filter(Boolean); instance.removeSelector ??= removeSelector?.length ? removeSelector : undefined; const targetSelector = ctx?.get('x-target-selector')?.split(', ').filter(Boolean); instance.targetSelector ??= targetSelector?.length ? targetSelector : undefined; const waitForSelector = ctx?.get('x-wait-for-selector')?.split(', ').filter(Boolean); instance.waitForSelector ??= (waitForSelector?.length ? waitForSelector : undefined) || instance.targetSelector; const overrideUserAgent = ctx?.get('x-user-agent') || undefined; instance.userAgent ??= overrideUserAgent; const engine = ctx?.get('x-engine'); if (engine) { instance.engine = engine; } if (instance.engine) { instance.engine = instance.engine.toLowerCase(); } if (instance.engine === 'vlm') { instance.engine = ENGINE_TYPE.BROWSER; instance.respondWith = CONTENT_FORMAT.VLM; } else if (instance.engine === 'readerlm-v2') { instance.engine = ENGINE_TYPE.AUTO; instance.respondWith = CONTENT_FORMAT.READER_LM; } const keepImgDataUrl = ctx?.get('x-keep-img-data-url'); if (keepImgDataUrl) { instance.keepImgDataUrl = Boolean(keepImgDataUrl); } const withIframe = ctx?.get('x-with-iframe'); if (withIframe) { instance.withIframe = withIframe.toLowerCase() === 'quoted' ? 'quoted' : Boolean(withIframe); } if (instance.withIframe) { instance.timeout ??= null; } const withShadowDom = ctx?.get('x-with-shadow-dom'); if (withShadowDom) { instance.withShadowDom = Boolean(withShadowDom); } if (instance.withShadowDom) { instance.timeout ??= null; } const cookies = []; const setCookieHeaders = (ctx?.get('x-set-cookie')?.split(', ') || instance.setCookies).filter(Boolean); if (Array.isArray(setCookieHeaders)) { for (const setCookie of setCookieHeaders) { cookies.push({ ...(0, set_cookie_parser_1.parseString)(setCookie, { decodeValues: true }), }); } } else if (setCookieHeaders && typeof setCookieHeaders === 'string') { cookies.push({ ...(0, set_cookie_parser_1.parseString)(setCookieHeaders, { decodeValues: true }), }); } instance.setCookies = cookies; const proxyUrl = ctx?.get('x-proxy-url'); instance.proxyUrl ??= proxyUrl || undefined; const proxy = ctx?.get('x-proxy'); instance.proxy ??= proxy || undefined; const robotsTxt = ctx?.get('x-robots-txt'); instance.robotsTxt ??= robotsTxt || undefined; const tokenBudget = ctx?.get('x-token-budget'); instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined; const baseMode = ctx?.get('x-base'); if (baseMode) { instance.base = baseMode; } const dnt = ctx?.get('dnt'); instance.doNotTrack ??= (parseInt(dnt || '') || null); const respondTiming = ctx?.get('x-respond-timing'); if (respondTiming) { instance.respondTiming ??= respondTiming; } if (instance.cacheTolerance) { instance.cacheTolerance = instance.cacheTolerance * 1000; } if (ctx) { instance.markdown ??= turndown_tweakable_options_1.TurnDownTweakableOptions.fromCtx(ctx); } return instance; } get presumedRespondTiming() { if (this.respondTiming) { return this.respondTiming; } if (this.timeout && this.timeout >= 20) { return RESPOND_TIMING.NETWORK_IDLE; } if (this.respondWith.includes('shot') || this.respondWith.includes('vlm')) { return RESPOND_TIMING.MEDIA_IDLE; } return RESPOND_TIMING.RESOURCE_IDLE; } isSnapshotAcceptableForEarlyResponse(snapshot) { if (this.waitForSelector?.length) { return false; } const presumedTiming = this.presumedRespondTiming; if (presumedTiming === RESPOND_TIMING.MEDIA_IDLE && snapshot.lastMediaResourceLoaded && snapshot.lastMutationIdle) { const now = Date.now(); if ((Math.max(snapshot.lastMediaResourceLoaded, snapshot.lastContentResourceLoaded || 0) + 500) < now) { return true; } } if ((this.respondWith.includes('vlm') || this.respondWith.includes('pageshot')) && !snapshot.pageshot) { return false; } if ((this.respondWith.includes('vlm') || this.respondWith.includes('screenshot')) && !snapshot.screenshot) { return false; } if (presumedTiming === RESPOND_TIMING.RESOURCE_IDLE && snapshot.lastContentResourceLoaded && snapshot.lastMutationIdle) { const now = Date.now(); if ((snapshot.lastContentResourceLoaded + 500) < now) { return true; } } if (this.injectFrameScript?.length || this.injectPageScript?.length) { return false; } if (presumedTiming === RESPOND_TIMING.VISIBLE_CONTENT && snapshot.parsed?.content) { return true; } if (presumedTiming === RESPOND_TIMING.HTML && snapshot.html) { return true; } if (presumedTiming === RESPOND_TIMING.NETWORK_IDLE) { return false; } if (presumedTiming === RESPOND_TIMING.MUTATION_IDLE && snapshot.lastMutationIdle) { return true; } if (this.respondWith.includes('lm')) { return false; } if (this.withIframe) { return false; } return !snapshot.isIntermediate; } isCacheQueryApplicable() { if (this.noCache) { return false; } if (this.cacheTolerance === 0) { return false; } if (this.setCookies?.length) { return false; } if (this.injectFrameScript?.length || this.injectPageScript?.length) { return false; } if (this.viewport) { return false; } return true; } isRequestingCompoundContentFormat() { return !CONTENT_FORMAT_VALUES.has(this.respondWith); } browserIsNotRequired() { if (this.respondTiming && ![RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes(this.respondTiming)) { return false; } if (this.respondWith.includes(CONTENT_FORMAT.PAGESHOT) || this.respondWith.includes(CONTENT_FORMAT.SCREENSHOT)) { return false; } if (this.injectFrameScript?.length || this.injectPageScript?.length) { return false; } if (this.waitForSelector?.length) { return false; } if (this.withIframe || this.withShadowDom) { return false; } if (this.viewport) { return false; } if (this.pdf) { return false; } if (this.html) { return false; } return true; } }; exports.CrawlerOptions = CrawlerOptions; __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "url", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "html", void 0); __decorate([ (0, civ_rpc_1.Prop)({ type: BASE_URL_MODE_VALUES, default: 'initial', }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "base", void 0); __decorate([ (0, civ_rpc_1.Prop)({ desc: 'Base64 encoded PDF.', type: [fancy_file_1.FancyFile, String] }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "pdf", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: CONTENT_FORMAT.CONTENT, type: [CONTENT_FORMAT, String] }), __metadata("design:type", String) ], CrawlerOptions.prototype, "respondWith", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, }), __metadata("design:type", Boolean) ], CrawlerOptions.prototype, "withGeneratedAlt", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: 'all', type: IMAGE_RETENTION_MODE_VALUES }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "retainImages", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "withLinksSummary", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, }), __metadata("design:type", Boolean) ], CrawlerOptions.prototype, "withImagesSummary", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, }), __metadata("design:type", Boolean) ], CrawlerOptions.prototype, "noCache", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "noGfm", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Number) ], CrawlerOptions.prototype, "cacheTolerance", void 0); __decorate([ (0, civ_rpc_1.Prop)({ arrayOf: String }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "targetSelector", void 0); __decorate([ (0, civ_rpc_1.Prop)({ arrayOf: String }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "waitForSelector", void 0); __decorate([ (0, civ_rpc_1.Prop)({ arrayOf: String }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "removeSelector", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, }), __metadata("design:type", Boolean) ], CrawlerOptions.prototype, "keepImgDataUrl", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, type: [String, Boolean] }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "withIframe", void 0); __decorate([ (0, civ_rpc_1.Prop)({ default: false, }), __metadata("design:type", Boolean) ], CrawlerOptions.prototype, "withShadowDom", void 0); __decorate([ (0, civ_rpc_1.Prop)({ arrayOf: String, }), __metadata("design:type", Array) ], CrawlerOptions.prototype, "setCookies", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "proxyUrl", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "proxy", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "userAgent", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "engine", void 0); __decorate([ (0, civ_rpc_1.Prop)({ arrayOf: String, }), __metadata("design:type", Array) ], CrawlerOptions.prototype, "injectPageScript", void 0); __decorate([ (0, civ_rpc_1.Prop)({ arrayOf: String, }), __metadata("design:type", Array) ], CrawlerOptions.prototype, "injectFrameScript", void 0); __decorate([ (0, civ_rpc_1.Prop)({ validate: (v) => v > 0 && v <= 180, type: Number, nullable: true, }), __metadata("design:type", Object) ], CrawlerOptions.prototype, "timeout", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "locale", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "referer", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Number) ], CrawlerOptions.prototype, "tokenBudget", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Viewport) ], CrawlerOptions.prototype, "viewport", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "instruction", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Object) ], CrawlerOptions.prototype, "jsonSchema", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", String) ], CrawlerOptions.prototype, "robotsTxt", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", Object) ], CrawlerOptions.prototype, "doNotTrack", void 0); __decorate([ (0, civ_rpc_1.Prop)(), __metadata("design:type", turndown_tweakable_options_1.TurnDownTweakableOptions) ], CrawlerOptions.prototype, "markdown", void 0); __decorate([ (0, civ_rpc_1.Prop)({ type: RESPOND_TIMING, }), __metadata("design:type", String) ], CrawlerOptions.prototype, "respondTiming", void 0); exports.CrawlerOptions = CrawlerOptions = __decorate([ (0, civ_rpc_1.Also)({ openapi: { operation: { parameters: { 'Accept': { description: `Specifies your preference for the response format.\n\n` + `Supported formats: \n` + `- text/event-stream\n` + `- application/json or text/json\n` + `- text/plain`, in: 'header', schema: { type: 'string' } }, 'X-Cache-Tolerance': { description: `Sets internal cache tolerance in seconds if this header is specified with a integer.`, in: 'header', schema: { type: 'string' } }, 'X-No-Cache': { description: `Ignores internal cache if this header is specified with a value.\n\nEquivalent to X-Cache-Tolerance: 0`, in: 'header', schema: { type: 'string' } }, 'X-Respond-With': { description: `Specifies the (non-default) form of the crawled data you prefer.\n\n` + `Supported formats: \n` + `- markdown\n` + `- html\n` + `- text\n` + `- pageshot\n` + `- screenshot\n` + `- content\n` + `- any combination of the above\n` + `- readerlm-v2\n` + `- vlm\n\n` + `Default: content\n`, in: 'header', schema: { type: 'string' } }, 'X-Wait-For-Selector': { description: `Specifies a CSS selector to wait for the appearance of such an element before returning.\n\n` + 'Example: `X-Wait-For-Selector: .content-block`\n', in: 'header', schema: { type: 'string' } }, 'X-Target-Selector': { description: `Specifies a CSS selector for return target instead of the full html.\n\n` + 'Implies `X-Wait-For-Selector: (same selector)`', in: 'header', schema: { type: 'string' } }, 'X-Remove-Selector': { description: `Specifies a CSS selector to remove elements from the full html.\n\n` + 'Example `X-Remove-Selector: nav`', in: 'header', schema: { type: 'string' } }, 'X-Keep-Img-Data-Url': { description: `Keep data-url as it instead of transforming them to object-url. (Only applicable when targeting markdown format)\n\n` + 'Example `X-Keep-Img-Data-Url: true`', in: 'header', schema: { type: 'string' } }, 'X-Proxy-Url': { description: `Specifies your custom proxy if you prefer to use one.\n\n` + `Supported protocols: \n` + `- http\n` + `- https\n` + `- socks4\n` + `- socks5\n\n` + `For authentication, https://user:pass@host:port`, in: 'header', schema: { type: 'string' } }, 'X-Proxy': { description: `Use a proxy server provided by us.\n\nOptionally specify two-letter country code.`, in: 'header', schema: { type: 'string' } }, 'X-Robots-Txt': { description: `Load and conform to the respective robot.txt on the target origin.\n\nOptionally specify a bot UA to check against.\n\n`, in: 'header', schema: { type: 'string' } }, 'DNT': { description: `When set to 1, prevent the result of this request to be cached in the system.\n\n`, in: 'header', schema: { type: 'string' } }, 'X-Set-Cookie': { description: `Sets cookie(s) to the headless browser for your request. \n\n` + `Syntax is the same with standard Set-Cookie`, in: 'header', schema: { type: 'string' } }, 'X-With-Generated-Alt': { description: `Enable automatic alt-text generating for images without an meaningful alt-text.\n\n` + `Note: Does not work when \`X-Respond-With\` is specified`, in: 'header', schema: { type: 'string' } }, 'X-With-Images-Summary': { description: `Enable dedicated summary section for images on the page.`, in: 'header', schema: { type: 'string' } }, 'X-With-links-Summary': { description: `Enable dedicated summary section for hyper links on the page.`, in: 'header', schema: { type: 'string' } }, 'X-Retain-Images': { description: `Image retention modes.\n\n` + `Supported modes: \n` + `- all: all images\n` + `- none: no images\n` + `- alt: only alt text\n` + `- all_p: all images and with generated alt text\n` + `- alt_p: only alt text and with generated alt\n\n`, in: 'header', schema: { type: 'string' } }, 'X-With-Iframe': { description: `Enable filling iframe contents into main. (violates standards)`, in: 'header', schema: { type: 'string' } }, 'X-With-Shadow-Dom': { description: `Enable filling shadow dom contents into main. (violates standards)`, in: 'header', schema: { type: 'string' } }, 'X-User-Agent': { description: `Override User-Agent.`, in: 'header', schema: { type: 'string' } }, 'X-Timeout': { description: `Specify timeout in seconds. Max 180.`, in: 'header', schema: { type: 'string' } }, 'X-Locale': { description: 'Specify browser locale for the page.', in: 'header', schema: { type: 'string' } }, 'X-Referer': { description: 'Specify referer for the page.', in: 'header', schema: { type: 'string' } }, 'X-Token-Budget': { description: 'Specify a budget in tokens.\n\nIf the resulting token cost exceeds the budget, the request is rejected.', in: 'header', schema: { type: 'string' } }, 'X-Respond-Timing': { description: `Explicitly specify the respond timing. One of the following:\n\n` + `- html: directly return unrendered HTML\n` + `- visible-content: return immediately when any content becomes available\n` + `- mutation-idle: wait for DOM mutations to settle and remain unchanged for at least 0.2s\n` + `- resource-idle: wait for no additional resources that would affect page logic and content has SUCCEEDED loading in 0.5s\n` + `- media-idle: wait for no additional resources, including media resources, has SUCCEEDED loading in 0.5s\n` + `- network-idle: wait for full load of webpage, also known as networkidle0.\n\n`, in: 'header', schema: { type: 'string' } }, 'X-Engine': { description: 'Specify the engine to use for crawling.\n\nSupported: browser, direct, cf-browser-rendering', in: 'header', schema: { type: 'string' } }, 'X-Base': { description: 'Select base modes of relative URLs.\n\nSupported: initial, final', in: 'header', schema: { type: 'string' } }, 'X-Md-Heading-Style': { description: 'Heading style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: setext, atx', in: 'header', schema: { type: 'string' } }, 'X-Md-Hr': { description: 'Hr text of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).', in: 'header', schema: { type: 'string' } }, 'X-Md-Bullet-List-Marker': { description: 'Bullet list marker of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: -, +, *', in: 'header', schema: { type: 'string' } }, 'X-Md-Em-Delimiter': { description: 'Em delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: _, *', in: 'header', schema: { type: 'string' } }, 'X-Md-Strong-Delimiter': { description: 'Strong delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: **, __', in: 'header', schema: { type: 'string' } }, 'X-Md-Link-Style': { description: 'Link style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: inlined, referenced, discarded', in: 'header', schema: { type: 'string' } }, 'X-Md-Link-Reference-Style': { description: 'Link reference style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: full, collapsed, shortcut, discarded', in: 'header', schema: { type: 'string' } }, } } } }) ], CrawlerOptions); class CrawlerOptionsHeaderOnly extends CrawlerOptions { static from(input) { const instance = super.from({ [civ_rpc_1.RPC_CALL_ENVIRONMENT]: Reflect.get(input, civ_rpc_1.RPC_CALL_ENVIRONMENT), }); return instance; } } exports.CrawlerOptionsHeaderOnly = CrawlerOptionsHeaderOnly; //# sourceMappingURL=crawler-options.js.map