build/api/searcher.js

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
    return function (target, key) { decorator(target, key, paramIndex); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
var _a, _b, _c, _d, _e, _f;
Object.defineProperty(exports, "__esModule", { value: true });
exports.SearcherHost = void 0;
const tsyringe_1 = require("tsyringe");
const civ_rpc_1 = require("civkit/civ-rpc");
const lang_1 = require("civkit/lang");
const hash_1 = require("civkit/hash");
const lodash_1 = __importDefault(require("lodash"));
const rate_limit_1 = require("../shared/services/rate-limit");
const crawler_1 = require("./crawler");
const crawler_options_1 = require("../dto/crawler-options");
const snapshot_formatter_1 = require("../services/snapshot-formatter");
const serper_search_1 = require("../services/serper-search");
const logger_1 = require("../services/logger");
const async_context_1 = require("../services/async-context");
const registry_1 = require("../services/registry");
const transform_server_event_stream_1 = require("../lib/transform-server-event-stream");
const jina_embeddings_auth_1 = require("../dto/jina-embeddings-auth");
const errors_1 = require("../services/errors");
// import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';
const misc_1 = require("../utils/misc");
const lru_cache_1 = require("lru-cache");
const api_roll_1 = require("../shared/db/api-roll");
const searched_1 = require("../db/searched");
// import { SerperSearchQueryParams, WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';
const internal_1 = require("../services/serp/internal");
const WORLD_COUNTRY_CODES = Object.keys(WORLD_COUNTRIES).map((x) => x.toLowerCase());
let SearcherHost = class SearcherHost extends civ_rpc_1.RPCHost {
    constructor(globalLogger, rateLimitControl, threadLocal, crawler, snapshotFormatter, serperGoogle, serperBing, jinaSerp) {
        super(...arguments);
        this.globalLogger = globalLogger;
        this.rateLimitControl = rateLimitControl;
        this.threadLocal = threadLocal;
        this.crawler = crawler;
        this.snapshotFormatter = snapshotFormatter;
        this.serperGoogle = serperGoogle;
        this.serperBing = serperBing;
        this.jinaSerp = jinaSerp;
        this.logger = this.globalLogger.child({ service: this.constructor.name });
        this.cacheRetentionMs = 1000 * 3600 * 24 * 7;
        this.cacheValidMs = 1000 * 3600;
        this.pageCacheToleranceMs = 1000 * 3600 * 24;
        this.reasonableDelayMs = 15_000;
        this.targetResultCount = 5;
        this.highFreqKeyCache = new lru_cache_1.LRUCache({
            max: 256,
            ttl: 60 * 60 * 1000,
            updateAgeOnGet: false,
            updateAgeOnHas: false,
        });
        this.batchedCaches = [];
        setInterval(() => {
            const thisBatch = this.batchedCaches;
            this.batchedCaches = [];
            if (!thisBatch.length) {
                return;
            }
            const batch = searched_1.SERPResult.DB.batch();
            for (const x of thisBatch) {
                batch.set(searched_1.SERPResult.COLLECTION.doc(), x.degradeForFireStore());
            }
            batch.commit()
                .then(() => {
                this.logger.debug(`Saved ${thisBatch.length} caches by batch`);
            })
                .catch((err) => {
                this.logger.warn(`Failed to cache search result in batch`, { err });
            });
        }, 1000 * 10 + Math.round(1000 * Math.random())).unref();
    }
    async init() {
        await this.dependencyReady();
        this.emit('ready');
    }
    async search(rpcReflect, ctx, auth, crawlerOptions, searchExplicitOperators, count, variant, searchEngine, num, gl, hl, location, page, fallback, q) {
        // We want to make our search API follow SERP schema, so we need to expose 'num' parameter.
        // Since we used 'count' as 'num' previously, we need to keep 'count' for old users.
        // Here we combine 'count' and 'num' to 'count' for the rest of the function.
        count = (num !== undefined ? num : count) ?? 10;
        const authToken = auth.bearerToken;
        let highFreqKey;
        if (authToken && this.highFreqKeyCache.has(authToken)) {
            highFreqKey = this.highFreqKeyCache.get(authToken);
            auth.user = highFreqKey.user;
            auth.uid = highFreqKey.user?.user_id;
        }
        const uid = await auth.solveUID();
        // Return content by default
        const crawlWithoutContent = crawlerOptions.respondWith.includes('no-content');
        const withFavicon = Boolean(ctx.get('X-With-Favicons'));
        this.threadLocal.set('collect-favicon', withFavicon);
        crawlerOptions.respondTiming ??= crawler_options_1.RESPOND_TIMING.VISIBLE_CONTENT;
        let chargeAmount = 0;
        const noSlashPath = decodeURIComponent(ctx.path).slice(1);
        if (!noSlashPath && !q) {
            const index = await this.crawler.getIndex(auth);
            if (!uid) {
                index.note = 'Authentication is required to use this endpoint. Please provide a valid API key via Authorization header.';
            }
            if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
                return index;
            }
            return (0, civ_rpc_1.assignTransferProtocolMeta)(`${index}`, { contentType: 'text/plain', envelope: null });
        }
        const user = await auth.assertUser();
        if (!(user.wallet.total_balance > 0)) {
            throw new errors_1.InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);
        }
        if (highFreqKey?.blockedUntil) {
            const now = new Date();
            const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf());
            if (blockedTimeRemaining > 0) {
                throw rate_limit_1.RateLimitTriggeredError.from({
                    message: `Per UID rate limit exceeded (async)`,
                    retryAfter: Math.ceil(blockedTimeRemaining / 1000),
                });
            }
        }
        const rateLimitPolicy = auth.getRateLimits(rpcReflect.name.toUpperCase()) || [
            parseInt(user.metadata?.speed_level) >= 2 ?
                rate_limit_1.RateLimitDesc.from({
                    occurrence: 1000,
                    periodSeconds: 60
                }) :
                rate_limit_1.RateLimitDesc.from({
                    occurrence: 100,
                    periodSeconds: 60
                })
        ];
        const apiRollPromise = this.rateLimitControl.simpleRPCUidBasedLimit(rpcReflect, uid, [rpcReflect.name.toUpperCase()], ...rateLimitPolicy);
        if (!highFreqKey) {
            // Normal path
            await apiRollPromise;
            if (rateLimitPolicy.some((x) => {
                const rpm = x.occurrence / (x.periodSeconds / 60);
                if (rpm >= 400) {
                    return true;
                }
                return false;
            })) {
                this.highFreqKeyCache.set(auth.bearerToken, {
                    user,
                });
            }
        }
        else {
            // High freq key path
            apiRollPromise.then(
            // Rate limit not triggered, make sure not blocking.
            () => {
                delete highFreqKey.blockedUntil;
            }, 
            // Rate limit triggered
            (err) => {
                if (!(err instanceof rate_limit_1.RateLimitTriggeredError)) {
                    return;
                }
                const now = Date.now();
                let tgtDate;
                if (err.retryAfterDate) {
                    tgtDate = err.retryAfterDate;
                }
                else if (err.retryAfter) {
                    tgtDate = new Date(now + err.retryAfter * 1000);
                }
                if (tgtDate) {
                    const dt = tgtDate.valueOf() - now;
                    highFreqKey.blockedUntil = tgtDate;
                    setTimeout(() => {
                        if (highFreqKey.blockedUntil === tgtDate) {
                            delete highFreqKey.blockedUntil;
                        }
                    }, dt).unref();
                }
            }).finally(async () => {
                // Always asynchronously update user(wallet);
                const user = await auth.getBrief().catch(() => undefined);
                if (user) {
                    highFreqKey.user = user;
                }
            });
        }
        rpcReflect.finally(async () => {
            if (chargeAmount) {
                auth.reportUsage(chargeAmount, `reader-${rpcReflect.name}`).catch((err) => {
                    this.logger.warn(`Unable to report usage for ${uid}`, { err: (0, lang_1.marshalErrorLike)(err) });
                });
                try {
                    const apiRoll = await apiRollPromise;
                    apiRoll.chargeAmount = chargeAmount;
                }
                catch (err) {
                    await this.rateLimitControl.record({
                        uid,
                        tags: [rpcReflect.name.toUpperCase()],
                        status: api_roll_1.API_CALL_STATUS.SUCCESS,
                        chargeAmount,
                    }).save().catch((err) => {
                        this.logger.warn(`Failed to save rate limit record`, { err: (0, lang_1.marshalErrorLike)(err) });
                    });
                }
            }
        });
        delete crawlerOptions.html;
        const crawlOpts = await this.crawler.configure(crawlerOptions);
        const searchQuery = searchExplicitOperators.addTo(q || noSlashPath);
        let fetchNum = count;
        if ((page ?? 1) === 1) {
            fetchNum = count > 10 ? 30 : 20;
        }
        let fallbackQuery;
        let chargeAmountScaler = 1;
        if (searchEngine === 'bing') {
            this.threadLocal.set('bing-preferred', true);
            chargeAmountScaler = 3;
        }
        if (variant !== 'web') {
            chargeAmountScaler = 5;
        }
        // Search with fallback logic if enabled
        const searchParams = {
            variant,
            provider: searchEngine,
            q: searchQuery,
            num: fetchNum,
            gl,
            hl,
            location,
            page,
        };
        const { results, query: successQuery, tryTimes } = await this.searchWithFallback(searchParams, fallback, crawlerOptions.noCache);
        chargeAmountScaler *= tryTimes;
        fallbackQuery = successQuery !== searchQuery ? successQuery : undefined;
        if (!results.length) {
            throw new civ_rpc_1.AssertionFailureError(`No search results available for query ${searchQuery}`);
        }
        if (crawlOpts.timeoutMs && crawlOpts.timeoutMs < 30_000) {
            delete crawlOpts.timeoutMs;
        }
        let lastScrapped;
        const targetResultCount = crawlWithoutContent ? count : count + 2;
        const trimmedResults = results.filter((x) => Boolean(x.link)).slice(0, targetResultCount).map((x) => this.mapToFinalResults(x));
        trimmedResults.toString = function () {
            let r = this.map((x, i) => x ? Reflect.apply(x.toString, x, [i]) : '').join('\n\n').trimEnd() + '\n';
            if (fallbackQuery) {
                r = `Fallback query: ${fallbackQuery}\n\n${r}`;
            }
            return r;
        };
        if (!crawlerOptions.respondWith.includes('no-content') &&
            ['html', 'text', 'shot', 'markdown', 'content'].some((x) => crawlerOptions.respondWith.includes(x))) {
            for (const x of trimmedResults) {
                x.content ??= '';
            }
        }
        const assigningOfGeneralMixins = Promise.allSettled(trimmedResults.map((x) => this.assignGeneralMixin(x)));
        let it;
        if (crawlWithoutContent || count === 0) {
            it = (0, misc_1.toAsyncGenerator)(trimmedResults);
            await assigningOfGeneralMixins;
        }
        else {
            it = this.fetchSearchResults(crawlerOptions.respondWith, trimmedResults, crawlOpts, crawler_options_1.CrawlerOptions.from({ ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs }), count);
        }
        if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) {
            const sseStream = new transform_server_event_stream_1.OutputServerEventStream();
            rpcReflect.return(sseStream);
            try {
                for await (const scrapped of it) {
                    if (!scrapped) {
                        continue;
                    }
                    if (rpcReflect.signal.aborted) {
                        break;
                    }
                    chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);
                    lastScrapped = scrapped;
                    if (fallbackQuery) {
                        sseStream.write({
                            event: 'meta',
                            data: { fallback: fallbackQuery },
                        });
                    }
                    sseStream.write({
                        event: 'data',
                        data: scrapped,
                    });
                }
            }
            catch (err) {
                this.logger.error(`Failed to collect search result for query ${searchQuery}`, { err: (0, lang_1.marshalErrorLike)(err) });
                sseStream.write({
                    event: 'error',
                    data: (0, lang_1.marshalErrorLike)(err),
                });
            }
            sseStream.end();
            return sseStream;
        }
        let earlyReturn = false;
        if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
            let earlyReturnTimer;
            const setEarlyReturnTimer = () => {
                if (earlyReturnTimer) {
                    return;
                }
                earlyReturnTimer = setTimeout(async () => {
                    if (!lastScrapped) {
                        return;
                    }
                    await assigningOfGeneralMixins;
                    chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
                    rpcReflect.return(lastScrapped);
                    earlyReturn = true;
                }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
            };
            for await (const scrapped of it) {
                lastScrapped = scrapped;
                if (rpcReflect.signal.aborted || earlyReturn) {
                    break;
                }
                if (lodash_1.default.some(scrapped, (x) => this.pageQualified(x))) {
                    setEarlyReturnTimer();
                }
                if (!this.searchResultsQualified(scrapped, count)) {
                    continue;
                }
                if (earlyReturnTimer) {
                    clearTimeout(earlyReturnTimer);
                }
                await assigningOfGeneralMixins;
                chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);
                return scrapped;
            }
            if (earlyReturnTimer) {
                clearTimeout(earlyReturnTimer);
            }
            if (!lastScrapped) {
                throw new civ_rpc_1.AssertionFailureError(`No content available for query ${searchQuery}`);
            }
            if (!earlyReturn) {
                await assigningOfGeneralMixins;
                chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
            }
            return lastScrapped;
        }
        let earlyReturnTimer;
        const setEarlyReturnTimer = () => {
            if (earlyReturnTimer) {
                return;
            }
            earlyReturnTimer = setTimeout(async () => {
                if (!lastScrapped) {
                    return;
                }
                await assigningOfGeneralMixins;
                chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
                rpcReflect.return((0, civ_rpc_1.assignTransferProtocolMeta)(`${lastScrapped}`, { contentType: 'text/plain', envelope: null }));
                earlyReturn = true;
            }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
        };
        for await (const scrapped of it) {
            lastScrapped = scrapped;
            if (rpcReflect.signal.aborted || earlyReturn) {
                break;
            }
            if (lodash_1.default.some(scrapped, (x) => this.pageQualified(x))) {
                setEarlyReturnTimer();
            }
            if (!this.searchResultsQualified(scrapped, count)) {
                continue;
            }
            if (earlyReturnTimer) {
                clearTimeout(earlyReturnTimer);
            }
            await assigningOfGeneralMixins;
            chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);
            return (0, civ_rpc_1.assignTransferProtocolMeta)(`${scrapped}`, { contentType: 'text/plain', envelope: null });
        }
        if (earlyReturnTimer) {
            clearTimeout(earlyReturnTimer);
        }
        if (!lastScrapped) {
            throw new civ_rpc_1.AssertionFailureError(`No content available for query ${searchQuery}`);
        }
        if (!earlyReturn) {
            await assigningOfGeneralMixins;
            chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
        }
        return (0, civ_rpc_1.assignTransferProtocolMeta)(`${lastScrapped}`, { contentType: 'text/plain', envelope: null });
    }
    /**
     * Search with fallback to progressively shorter queries if no results found
     * @param params Search parameters
     * @param useFallback Whether to use the fallback mechanism
     * @param noCache Whether to bypass cache
     * @returns Search response and the successful query
     */
    async searchWithFallback(params, useFallback = false, noCache = false) {
        // Try original query first
        const originalQuery = params.q;
        const containsRTL = /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test(originalQuery);
        // Extract results based on variant
        let tryTimes = 1;
        const results = await this.cachedSearch(params.variant, params, noCache);
        if (results.length || !useFallback) {
            return { results, query: params.q, tryTimes };
        }
        let queryTerms = originalQuery.split(/\s+/);
        const lastResort = containsRTL ? queryTerms.slice(queryTerms.length - 2) : queryTerms.slice(0, 2);
        this.logger.info(`No results for "${originalQuery}", trying fallback queries`);
        let terms = [];
        // fallback n times
        const n = 4;
        while (tryTimes < n) {
            const delta = Math.ceil(queryTerms.length / n) * tryTimes;
            terms = containsRTL ? queryTerms.slice(delta) : queryTerms.slice(0, queryTerms.length - delta);
            const query = terms.join(' ');
            if (!query) {
                break;
            }
            tryTimes += 1;
            this.logger.info(`Retrying search with fallback query: "${query}"`);
            const fallbackParams = { ...params, q: query };
            const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache);
            if (fallbackResults.length > 0) {
                return { results: fallbackResults, query: fallbackParams.q, tryTimes };
            }
        }
        if (terms.length > lastResort.length) {
            const query = lastResort.join(' ');
            this.logger.info(`Retrying search with fallback query: "${query}"`);
            const fallbackParams = { ...params, q: query };
            tryTimes += 1;
            const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache);
            if (fallbackResults.length > 0) {
                return { results: fallbackResults, query, tryTimes };
            }
        }
        return { results, query: originalQuery, tryTimes };
    }
    async *fetchSearchResults(mode, searchResults, options, crawlerOptions, count) {
        if (!searchResults) {
            return;
        }
        const urls = searchResults.map((x) => new URL(x.url));
        const snapshotMap = new WeakMap();
        for await (const scrapped of this.crawler.scrapMany(urls, options, crawlerOptions)) {
            const mapped = scrapped.map((x, i) => {
                if (!x) {
                    return {};
                }
                if (snapshotMap.has(x)) {
                    return snapshotMap.get(x);
                }
                return this.crawler.formatSnapshotWithPDFSideLoad(mode, x, urls[i], undefined, options).then((r) => {
                    snapshotMap.set(x, r);
                    return r;
                }).catch((err) => {
                    this.logger.error(`Failed to format snapshot for ${urls[i].href}`, { err: (0, lang_1.marshalErrorLike)(err) });
                    return {};
                });
            });
            const resultArray = await Promise.all(mapped);
            for (const [i, v] of resultArray.entries()) {
                if (v) {
                    Object.assign(searchResults[i], v);
                }
            }
            yield this.reOrganizeSearchResults(searchResults, count);
        }
    }
    reOrganizeSearchResults(searchResults, count) {
        const targetResultCount = count || this.targetResultCount;
        const [qualifiedPages, unqualifiedPages] = lodash_1.default.partition(searchResults, (x) => this.pageQualified(x));
        const acceptSet = new Set(qualifiedPages);
        const n = targetResultCount - qualifiedPages.length;
        for (const x of unqualifiedPages.slice(0, n >= 0 ? n : 0)) {
            acceptSet.add(x);
        }
        const filtered = searchResults.filter((x) => acceptSet.has(x)).slice(0, targetResultCount);
        const resultArray = filtered;
        resultArray.toString = searchResults.toString;
        return resultArray;
    }
    assignChargeAmount(formatted, num, scaler, fallbackQuery) {
        let contentCharge = 0;
        for (const x of formatted) {
            const itemAmount = this.crawler.assignChargeAmount(x) || 0;
            if (!itemAmount) {
                continue;
            }
            contentCharge += itemAmount;
        }
        const numCharge = Math.ceil(formatted.length / 10) * 10000 * scaler;
        const final = Math.max(contentCharge, numCharge);
        if (final === numCharge) {
            for (const x of formatted) {
                x.usage = { tokens: Math.ceil(numCharge / formatted.length) };
            }
        }
        const metadata = { usage: { tokens: final } };
        if (fallbackQuery) {
            metadata.fallback = fallbackQuery;
        }
        (0, civ_rpc_1.assignMeta)(formatted, metadata);
        return final;
    }
    pageQualified(formattedPage) {
        return formattedPage.title &&
            formattedPage.content ||
            formattedPage.screenshotUrl ||
            formattedPage.pageshotUrl ||
            formattedPage.text ||
            formattedPage.html;
    }
    searchResultsQualified(results, targetResultCount = this.targetResultCount) {
        return lodash_1.default.every(results, (x) => this.pageQualified(x)) && results.length >= targetResultCount;
    }
    async getFavicon(domain) {
        const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`;
        try {
            const response = await fetch(url);
            if (!response.ok) {
                return '';
            }
            const ab = await response.arrayBuffer();
            const buffer = Buffer.from(ab);
            const base64 = buffer.toString('base64');
            return `data:image/png;base64,${base64}`;
        }
        catch (error) {
            this.logger.warn(`Failed to get favicon base64 string`, { err: (0, lang_1.marshalErrorLike)(error) });
            return '';
        }
    }
    *iterProviders(preference, variant) {
        if (preference === 'bing') {
            yield this.serperBing;
            yield variant === 'web' ? this.jinaSerp : this.serperGoogle;
            yield this.serperGoogle;
            return;
        }
        if (preference === 'google') {
            yield variant === 'web' ? this.jinaSerp : this.serperGoogle;
            yield this.serperGoogle;
            yield this.serperGoogle;
            return;
        }
        yield variant === 'web' ? this.jinaSerp : this.serperGoogle;
        yield this.serperGoogle;
        yield this.serperGoogle;
    }
    async cachedSearch(variant, query, noCache) {
        const queryDigest = (0, hash_1.objHashMd5B64Of)({ ...query, variant });
        const provider = query.provider;
        Reflect.deleteProperty(query, 'provider');
        let cache;
        if (!noCache) {
            cache = (await searched_1.SERPResult.fromFirestoreQuery(searched_1.SERPResult.COLLECTION.where('queryDigest', '==', queryDigest)
                .orderBy('createdAt', 'desc')
                .limit(1)))[0];
            if (cache) {
                const age = Date.now() - cache.createdAt.valueOf();
                const stale = cache.createdAt.valueOf() < (Date.now() - this.cacheValidMs);
                this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`, {
                    query, digest: queryDigest, age, stale
                });
                if (!stale) {
                    return cache.response;
                }
            }
        }
        try {
            let r;
            let lastError;
            outerLoop: for (const client of this.iterProviders(provider, variant)) {
                const t0 = Date.now();
                try {
                    switch (variant) {
                        case 'images': {
                            r = await Reflect.apply(client.imageSearch, client, [query]);
                            break;
                        }
                        case 'news': {
                            r = await Reflect.apply(client.newsSearch, client, [query]);
                            break;
                        }
                        case 'web':
                        default: {
                            r = await Reflect.apply(client.webSearch, client, [query]);
                            break;
                        }
                    }
                    const dt = Date.now() - t0;
                    this.logger.info(`Search took ${dt}ms, ${client.constructor.name}(${variant})`, { searchDt: dt, variant, client: client.constructor.name });
                    break outerLoop;
                }
                catch (err) {
                    lastError = err;
                    const dt = Date.now() - t0;
                    this.logger.warn(`Failed to do ${variant} search using ${client.constructor.name}`, { err, variant, searchDt: dt, });
                }
            }
            if (r?.length) {
                const nowDate = new Date();
                const record = searched_1.SERPResult.from({
                    query,
                    queryDigest,
                    response: r,
                    createdAt: nowDate,
                    expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs)
                });
                this.batchedCaches.push(record);
            }
            else if (lastError) {
                throw lastError;
            }
            return r;
        }
        catch (err) {
            if (cache) {
                this.logger.warn(`Failed to fetch search result, but a stale cache is available. falling back to stale cache`, { err: (0, lang_1.marshalErrorLike)(err) });
                return cache.response;
            }
            throw err;
        }
    }
    mapToFinalResults(input) {
        const whitelistedProps = [
            'imageUrl', 'imageWidth', 'imageHeight', 'source', 'date', 'siteLinks'
        ];
        const result = {
            title: input.title,
            url: input.link,
            description: Reflect.get(input, 'snippet'),
            ...lodash_1.default.pick(input, whitelistedProps),
        };
        return result;
    }
    async assignGeneralMixin(result) {
        const collectFavicon = this.threadLocal.get('collect-favicon');
        if (collectFavicon && result.url) {
            const url = new URL(result.url);
            Reflect.set(result, 'favicon', await this.getFavicon(url.origin));
        }
        Object.setPrototypeOf(result, searchResultProto);
    }
};
exports.SearcherHost = SearcherHost;
__decorate([
    (0, registry_1.Method)({
        name: 'searchIndex',
        ext: {
            http: {
                action: ['get', 'post'],
                path: '/search'
            }
        },
        tags: ['search'],
        returnType: [String, transform_server_event_stream_1.OutputServerEventStream],
    }),
    (0, registry_1.Method)({
        ext: {
            http: {
                action: ['get', 'post'],
                path: '::q'
            }
        },
        tags: ['search'],
        returnType: [String, transform_server_event_stream_1.OutputServerEventStream, civ_rpc_1.RawString],
    }),
    __param(0, (0, registry_1.RPCReflect)()),
    __param(1, (0, registry_1.Ctx)()),
    __param(5, (0, registry_1.Param)('count', { validate: (v) => v >= 0 && v <= 20 })),
    __param(6, (0, registry_1.Param)('type', { type: new Set(['web', 'images', 'news']), default: 'web' })),
    __param(7, (0, registry_1.Param)('provider', { type: new Set(['google', 'bing']), default: 'google' })),
    __param(8, (0, registry_1.Param)('num', { validate: (v) => v >= 0 && v <= 20 })),
    __param(9, (0, registry_1.Param)('gl', { validate: (v) => WORLD_COUNTRY_CODES.includes(v?.toLowerCase()) })),
    __param(10, (0, registry_1.Param)('hl', { validate: (v) => WORLD_LANGUAGES.some(l => l.code === v) })),
    __param(11, (0, registry_1.Param)('location')),
    __param(12, (0, registry_1.Param)('page')),
    __param(13, (0, registry_1.Param)('fallback', { type: Boolean, default: true })),
    __param(14, (0, registry_1.Param)('q')),
    __metadata("design:type", Function),
    __metadata("design:paramtypes", [typeof (_d = typeof civ_rpc_1.RPCReflection !== "undefined" && civ_rpc_1.RPCReflection) === "function" ? _d : Object, typeof (_e = typeof registry_1.Context !== "undefined" && registry_1.Context) === "function" ? _e : Object, jina_embeddings_auth_1.JinaEmbeddingsAuthDTO,
        crawler_options_1.CrawlerOptions, typeof (_f = typeof serper_search_1.GoogleSearchExplicitOperatorsDto !== "undefined" && serper_search_1.GoogleSearchExplicitOperatorsDto) === "function" ? _f : Object, Number, String, String, Number, String, String, String, Number, Boolean, String]),
    __metadata("design:returntype", Promise)
], SearcherHost.prototype, "search", null);
exports.SearcherHost = SearcherHost = __decorate([
    (0, tsyringe_1.singleton)(),
    __metadata("design:paramtypes", [logger_1.GlobalLogger, typeof (_a = typeof rate_limit_1.RateLimitControl !== "undefined" && rate_limit_1.RateLimitControl) === "function" ? _a : Object, async_context_1.AsyncLocalContext,
        crawler_1.CrawlerHost,
        snapshot_formatter_1.SnapshotFormatter, typeof (_b = typeof SerperGoogleSearchService !== "undefined" && SerperGoogleSearchService) === "function" ? _b : Object, typeof (_c = typeof SerperBingSearchService !== "undefined" && SerperBingSearchService) === "function" ? _c : Object, internal_1.InternalJinaSerpService])
], SearcherHost);
const dataItems = [
    { key: 'title', label: 'Title' },
    { key: 'source', label: 'Source' },
    { key: 'url', label: 'URL Source' },
    { key: 'imageUrl', label: 'Image URL' },
    { key: 'description', label: 'Description' },
    { key: 'publishedTime', label: 'Published Time' },
    { key: 'imageWidth', label: 'Image Width' },
    { key: 'imageHeight', label: 'Image Height' },
    { key: 'date', label: 'Date' },
    { key: 'favicon', label: 'Favicon' },
];
const searchResultProto = {
    toString(i) {
        const chunks = [];
        for (const item of dataItems) {
            const v = Reflect.get(this, item.key);
            if (typeof v !== 'undefined') {
                if (i === undefined) {
                    chunks.push(`[${item.label}]: ${v}`);
                }
                else {
                    chunks.push(`[${i + 1}] ${item.label}: ${v}`);
                }
            }
        }
        if (this.content) {
            chunks.push(`\n${this.content}`);
        }
        if (this.images) {
            const imageSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Images:`];
            for (const [k, v] of Object.entries(this.images)) {
                imageSummaryChunks.push(`- ![${k}](${v})`);
            }
            if (imageSummaryChunks.length === 1) {
                imageSummaryChunks.push('This page does not seem to contain any images.');
            }
            chunks.push(imageSummaryChunks.join('\n'));
        }
        if (this.links) {
            const linkSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Links/Buttons:`];
            if (Array.isArray(this.links)) {
                for (const [k, v] of this.links) {
                    linkSummaryChunks.push(`- [${k}](${v})`);
                }
            }
            else {
                for (const [k, v] of Object.entries(this.links)) {
                    linkSummaryChunks.push(`- [${k}](${v})`);
                }
            }
            if (linkSummaryChunks.length === 1) {
                linkSummaryChunks.push('This page does not seem to contain any buttons/links.');
            }
            chunks.push(linkSummaryChunks.join('\n'));
        }
        return chunks.join('\n');
    }
};
//# sourceMappingURL=searcher.js.map