import axios from "axios"; import { getEnv } from "../../shared/config/env"; import { withRetry, isCircuitOpen, recordFailure, recordSuccess } from "../../shared/utils/retry"; import { serperLimiter } from "../../shared/utils/rate-limiter"; import { logger } from "../../shared/utils/logger"; const PROVIDER = "serper"; export interface SerperResult { title: string; link: string; snippet: string; domain: string; } /** * Searches Google via Serper.dev API. * Builds targeted queries to find companies matching ICP in a given region. */ export async function searchCompanies( region: string, industry: string, keywords: string[], page = 1 ): Promise { if (isCircuitOpen(PROVIDER)) { logger.warn({ provider: PROVIDER }, "Circuit open — skipping Serper call"); return []; } await serperLimiter.consume(PROVIDER); const queries = buildQueries(region, industry, keywords); const results: SerperResult[] = []; for (const query of queries) { try { const data = await withRetry( () => callSerper(query, page), { provider: PROVIDER } ); results.push(...data); recordSuccess(PROVIDER); } catch (err) { recordFailure(PROVIDER); logger.error({ query, err }, "Serper search failed"); } } // Deduplicate by domain const seen = new Set(); return results.filter((r) => { if (seen.has(r.domain)) return false; seen.add(r.domain); return true; }); } async function callSerper(query: string, page: number): Promise { const env = getEnv(); const response = await axios.post( "https://google.serper.dev/search", { q: query, num: 10, page }, { headers: { "X-API-KEY": env.SERPER_API_KEY, "Content-Type": "application/json", }, timeout: 10_000, } ); const organic = response.data?.organic ?? []; return organic.map((item: { title: string; link: string; snippet: string }) => ({ title: item.title, link: item.link, snippet: item.snippet, domain: extractDomain(item.link), })); } function buildQueries(region: string, industry: string, keywords: string[]): string[] { // Precision queries — each targets a specific pain+industry+region combo const regionLabel = REGION_LABELS[region] ?? region; return [ `"${industry}" company "${regionLabel}" "50 employees" OR "100 employees" OR "200 employees" automation`, `${industry} business ${regionLabel} site:linkedin.com/company`, `"${industry}" "${regionLabel}" "digital transformation" OR "AI" OR "automation" company`, `${keywords[0]} ${keywords[1] ?? ""} company ${regionLabel} -job -careers`, ].filter(Boolean); } function extractDomain(url: string): string { try { return new URL(url).hostname.replace(/^www\./, ""); } catch { return url; } } const REGION_LABELS: Record = { US: "United States", UK: "United Kingdom", AU: "Australia", UAE: "Dubai", SA: "Saudi Arabia", SG: "Singapore", };