Download src/lib/ddg.ts from AnesKAM/gen: direct link, hf CLI and curl.
- Browser
- Download file 35.5 kB
-
https://huggingface.co/spaces/AnesKAM/gen/resolve/main/src/lib/ddg.ts
- Command line
-
hf download hf://spaces/AnesKAM/gen/src/lib/ddg.ts
-
curl -L -o ddg.ts https://huggingface.co/spaces/AnesKAM/gen/resolve/main/src/lib/ddg.ts
35.5 kB
| import * as cheerio from "cheerio"; | |
| import type { | |
| ImageResult, | |
| InstantAnswer, | |
| NewsResult, | |
| RelatedItem, | |
| SearchResult, | |
| SearchTab, | |
| VideoResult, | |
| WebResult, | |
| } from "@/lib/types"; | |
| import { decodeHtmlEntities, domainOf, isHttpUrl, sleep } from "@/lib/utils"; | |
| /* | |
| * DuckDuckGo client with real pagination. | |
| * | |
| * All -> links.duckduckgo.com/d.js (10 results, then 15 per page, follows the `n` link) | |
| * News -> duckduckgo.com/news.js (~30 per page, follows `next`) | |
| * Videos -> duckduckgo.com/v.js (60 per page, follows `next`) | |
| * Photos -> duckduckgo.com/i.js (100 per page, follows `next`) | |
| * | |
| * Nothing is truncated: every page is returned exactly as DuckDuckGo serves it, | |
| * together with an opaque cursor for the page after it. | |
| */ | |
| const DDG = "https://duckduckgo.com"; | |
| const LINKS = "https://links.duckduckgo.com"; | |
| const HTML_ENDPOINT = "https://html.duckduckgo.com/html/"; | |
| const USER_AGENTS = [ | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", | |
| "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15", | |
| "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0", | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", | |
| ]; | |
| type Json = Record<string, unknown>; | |
| type Vertical = Exclude<SearchTab, "all">; | |
| export interface PageResult { | |
| results: SearchResult[]; | |
| related: RelatedItem[]; | |
| /** 1-based DuckDuckGo page number. */ | |
| page: number; | |
| /** Cursor for the following page; null when DuckDuckGo has no more. */ | |
| next: string | null; | |
| /** The upstream request failed or was challenged; `next` then repeats this page. */ | |
| degraded: boolean; | |
| fallback: string | null; | |
| /** Set when results are for a simplified query (see `relaxQuery`). */ | |
| shownFor?: string | null; | |
| } | |
| /* ------------------------------------------------------------------ cursors */ | |
| export interface Cursor { | |
| /** | |
| * d = d.js web page, q = related/expanded DDG web page (when s=10+ is challenged), | |
| * j = JSON vertical (news/videos/photos), | |
| * c = Wikimedia Commons photo fallback, w = Wikipedia web fallback. | |
| */ | |
| k: "d" | "q" | "j" | "c" | "w"; | |
| /** Page number this cursor points at. */ | |
| p: number; | |
| /** User-agent index, kept stable for the whole chain. */ | |
| u: number; | |
| /** Upstream "next" reference (path + query, sub-query, or an offset). */ | |
| n: string; | |
| /** vqd token for JSON verticals. */ | |
| v?: string; | |
| /** Queued sub-queries to seamlessly continue pagination if DDG challenges s=10+. */ | |
| alt?: string[]; | |
| } | |
| function encodeCursor(cursor: Cursor): string { | |
| return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); | |
| } | |
| /** Returns null for anything that is not a well-formed cursor. */ | |
| export function decodeCursor(raw: string | null | undefined): Cursor | null { | |
| if (!raw || raw.length > 8000) return null; | |
| try { | |
| const c = JSON.parse(Buffer.from(raw, "base64url").toString("utf8")) as Partial<Cursor> | null; | |
| if (!c || typeof c !== "object") return null; | |
| if (c.k !== "d" && c.k !== "q" && c.k !== "j" && c.k !== "c" && c.k !== "w") return null; | |
| if (typeof c.p !== "number" || !Number.isInteger(c.p) || c.p < 2 || c.p > 1000) return null; | |
| if (typeof c.u !== "number" || !Number.isInteger(c.u) || c.u < 0 || c.u > 50) return null; | |
| if (typeof c.n !== "string" || c.n.length === 0 || c.n.length > 4000) return null; | |
| if (c.v !== undefined && (typeof c.v !== "string" || !/^[0-9A-Za-z-]{5,200}$/.test(c.v))) return null; | |
| const alt = Array.isArray(c.alt) | |
| ? c.alt.filter((x): x is string => typeof x === "string" && x.length > 0 && x.length <= 200).slice(0, 12) | |
| : undefined; | |
| return { | |
| k: c.k, | |
| p: c.p, | |
| u: c.u, | |
| n: c.n, | |
| ...(c.v ? { v: c.v } : {}), | |
| ...(alt && alt.length > 0 ? { alt } : {}), | |
| }; | |
| } catch { | |
| return null; | |
| } | |
| } | |
| /** | |
| * Cursors come back from the browser, so the upstream reference is only ever | |
| * resolved against a fixed DuckDuckGo origin + path. Anything else is rejected. | |
| */ | |
| function pinnedUrl(reference: string, origin: string, path: string): string | null { | |
| try { | |
| const url = new URL(reference, `${origin}/`); | |
| if (url.origin !== origin || url.pathname !== path) return null; | |
| return url.toString(); | |
| } catch { | |
| return null; | |
| } | |
| } | |
| /* ------------------------------------------------------------------ helpers */ | |
| function str(value: unknown): string { | |
| return typeof value === "string" ? value : ""; | |
| } | |
| function httpOrNull(value: unknown): string | null { | |
| return isHttpUrl(value) ? value : null; | |
| } | |
| function clean(value: unknown): string { | |
| if (typeof value !== "string" || !value) return ""; | |
| const text = value | |
| .replace(/<\/?(?:b|strong|em|i|span|mark|u)\b[^>]*>/gi, "") | |
| .replace(/<[^>]*>/g, " "); | |
| return decodeHtmlEntities(text).replace(/\s+/g, " ").trim(); | |
| } | |
| function toIso(value: unknown): string | null { | |
| if (typeof value === "number" && value > 0) { | |
| const date = new Date(value > 1e12 ? value : value * 1000); | |
| return Number.isNaN(date.getTime()) ? null : date.toISOString(); | |
| } | |
| if (typeof value !== "string" || !value.trim()) return null; | |
| // DuckDuckGo emits "2024-03-02T22:30:08.0000000" (7 fraction digits, no zone). | |
| let text = value.trim().replace(/(\.\d{3})\d+/, "$1"); | |
| if (/^\d{4}-\d{2}-\d{2}T[\d:.]+$/.test(text)) text += "Z"; | |
| const date = new Date(text); | |
| return Number.isNaN(date.getTime()) ? null : date.toISOString(); | |
| } | |
| /** Arabic queries get the Arabic region; other non-Latin scripts search worldwide. */ | |
| export function regionFor(query: string): string { | |
| if (/[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF]/.test(query)) return "xa-ar"; | |
| if (/[\u0400-\u04FF\u0590-\u05FF\u0900-\u0DFF\u0E00-\u0E7F\u3040-\u30FF\u4E00-\u9FFF\uAC00-\uD7AF]/.test(query)) { | |
| return "wt-wt"; | |
| } | |
| return "us-en"; | |
| } | |
| function ended(page: number): PageResult { | |
| return { results: [], related: [], page, next: null, degraded: false, fallback: null }; | |
| } | |
| function failed(page: number, retryCursor: Cursor | null): PageResult { | |
| return { | |
| results: [], | |
| related: [], | |
| page, | |
| next: retryCursor ? encodeCursor(retryCursor) : null, | |
| degraded: true, | |
| fallback: null, | |
| }; | |
| } | |
| async function ddgFetch( | |
| url: string, | |
| ua: number, | |
| accept: string, | |
| referer = `${DDG}/`, | |
| ): Promise<{ status: number; text: string } | null> { | |
| try { | |
| const res = await fetch(url, { | |
| headers: { | |
| "user-agent": USER_AGENTS[ua % USER_AGENTS.length], | |
| accept, | |
| "accept-language": "en-US,en;q=0.9", | |
| referer, | |
| }, | |
| cache: "no-store", | |
| signal: AbortSignal.timeout(12_000), | |
| }); | |
| return { status: res.status, text: await res.text() }; | |
| } catch { | |
| return null; | |
| } | |
| } | |
| /* ------------------------------------------------- session bootstrap (vqd) */ | |
| interface Bootstrap { | |
| vqd: string; | |
| /** Fully signed first-page d.js URL taken from DuckDuckGo's own page. */ | |
| djs: string | null; | |
| ua: number; | |
| at: number; | |
| } | |
| const BOOT_TTL_MS = 8 * 60 * 1000; | |
| const bootCache = new Map<string, Bootstrap>(); | |
| const bootInflight = new Map<string, Promise<Bootstrap | null>>(); | |
| /** | |
| * Loads the DuckDuckGo results page once per query to obtain the `vqd` token | |
| * (required by every vertical) and the signed d.js URL. Cached briefly so that | |
| * switching tabs or fetching inline videos on the same query reuses one token. | |
| */ | |
| async function bootstrap(query: string, region: string, fresh = false): Promise<Bootstrap | null> { | |
| const key = `${region}::${query}`; | |
| const hit = bootCache.get(key); | |
| if (!fresh && hit && Date.now() - hit.at < BOOT_TTL_MS) return hit; | |
| if (!fresh) { | |
| const pending = bootInflight.get(key); | |
| if (pending) return pending; | |
| } | |
| const job = (async (): Promise<Bootstrap | null> => { | |
| for (let attempt = 0; attempt < 3; attempt++) { | |
| const ua = attempt; | |
| const res = await ddgFetch( | |
| `${DDG}/?q=${encodeURIComponent(query)}&kl=${region}&ia=web`, | |
| ua, | |
| "text/html,application/xhtml+xml", | |
| ); | |
| if (res && res.status === 200) { | |
| const vqd = | |
| res.text.match(/vqd="([0-9A-Za-z-]+)"/)?.[1] ?? res.text.match(/[?&]vqd=([0-9][0-9A-Za-z-]+)/)?.[1]; | |
| if (vqd) { | |
| const djsRaw = res.text.match(/src="(https:\/\/links\.duckduckgo\.com\/d\.js\?[^"]+)"/)?.[1]; | |
| const djs = djsRaw ? pinnedUrl(decodeHtmlEntities(djsRaw), LINKS, "/d.js") : null; | |
| const boot: Bootstrap = { vqd, djs, ua, at: Date.now() }; | |
| bootCache.set(key, boot); | |
| if (bootCache.size > 300) { | |
| const oldest = bootCache.keys().next().value; | |
| if (oldest !== undefined) bootCache.delete(oldest); | |
| } | |
| return boot; | |
| } | |
| } | |
| await sleep(350 + attempt * 450); | |
| } | |
| bootCache.delete(key); | |
| return null; | |
| })().finally(() => { | |
| bootInflight.delete(key); | |
| }); | |
| if (!fresh) bootInflight.set(key, job); | |
| return job; | |
| } | |
| /* ------------------------------------------------------- web results: d.js */ | |
| /** Pulls a JSON literal that follows `marker` out of DuckDuckGo's JS payload. */ | |
| function extractBalanced(text: string, marker: string, open: string, close: string): string | null { | |
| const at = text.indexOf(marker); | |
| if (at < 0) return null; | |
| const start = text.indexOf(open, at + marker.length); | |
| if (start < 0 || text.slice(at + marker.length, start).trim() !== "") return null; | |
| let depth = 0; | |
| let inString = false; | |
| let escaped = false; | |
| for (let i = start; i < text.length; i++) { | |
| const ch = text[i]; | |
| if (inString) { | |
| if (escaped) escaped = false; | |
| else if (ch === "\\") escaped = true; | |
| else if (ch === '"') inString = false; | |
| continue; | |
| } | |
| if (ch === '"') inString = true; | |
| else if (ch === open) depth++; | |
| else if (ch === close) { | |
| depth--; | |
| if (depth === 0) return text.slice(start, i + 1); | |
| } | |
| } | |
| return null; | |
| } | |
| function isDdgInternal(url: string): boolean { | |
| try { | |
| const u = new URL(url); | |
| return /(^|\.)duckduckgo\.com$/.test(u.hostname) && (u.pathname === "/y.js" || u.pathname.startsWith("/l/")); | |
| } catch { | |
| return true; | |
| } | |
| } | |
| function parseRelated(text: string): RelatedItem[] { | |
| const raw = extractBalanced(text, "DDG.duckbar.loadModule('related_searches',", "{", "}"); | |
| if (!raw) return []; | |
| try { | |
| const data = JSON.parse(raw) as { results?: Json[] }; | |
| const out: RelatedItem[] = []; | |
| const seen = new Set<string>(); | |
| for (const row of Array.isArray(data.results) ? data.results : []) { | |
| const query = clean(row.text); | |
| const key = query.toLowerCase(); | |
| if (!query || seen.has(key)) continue; | |
| seen.add(key); | |
| out.push({ text: clean(row.display_text) || query, query }); | |
| if (out.length >= 10) break; | |
| } | |
| return out; | |
| } catch { | |
| return []; | |
| } | |
| } | |
| interface DjsPage { | |
| results: WebResult[]; | |
| next: string | null; | |
| related: RelatedItem[]; | |
| /** User-agent index of the session that produced page 1. */ | |
| ua?: number; | |
| } | |
| function parseDjs(text: string): DjsPage | null { | |
| const raw = extractBalanced(text, "DDG.pageLayout.load('d',", "[", "]"); | |
| if (!raw) return null; | |
| let items: unknown; | |
| try { | |
| items = JSON.parse(raw); | |
| } catch { | |
| return null; | |
| } | |
| if (!Array.isArray(items)) return null; | |
| const results: WebResult[] = []; | |
| const seen = new Set<string>(); | |
| let next: string | null = null; | |
| for (const item of items as Json[]) { | |
| if (!item || typeof item !== "object") continue; | |
| if (typeof item.n === "string" && item.n) { | |
| next = item.n; | |
| continue; | |
| } | |
| const url = str(item.u) || str(item.c); | |
| const title = clean(item.t); | |
| if (!title || !isHttpUrl(url) || seen.has(url) || isDdgInternal(url)) continue; | |
| seen.add(url); | |
| results.push({ | |
| kind: "web", | |
| title, | |
| url, | |
| snippet: clean(item.a), | |
| displayUrl: clean(item.d) || domainOf(url), | |
| date: toIso(item.e), | |
| }); | |
| } | |
| return { results, next, related: parseRelated(text) }; | |
| } | |
| type DjsOutcome = | |
| | { kind: "ok"; page: DjsPage } | |
| /** DuckDuckGo's anomaly detection refused the request (rate limit, or its ~200-result depth cap). */ | |
| | { kind: "blocked" } | |
| /** DuckDuckGo answered with a browser challenge instead of results (quoted / operator queries). */ | |
| | { kind: "challenge" } | |
| /** Network failure or server error: the only case worth retrying immediately. */ | |
| | { kind: "error" }; | |
| async function fetchDjs(url: string, ua: number): Promise<DjsOutcome> { | |
| const res = await ddgFetch(url, ua, "*/*"); | |
| if (!res || res.status >= 500) return { kind: "error" }; | |
| if (res.status === 403 || res.status === 429 || res.text.includes("anomalyDetectionBlock")) { | |
| return { kind: "blocked" }; | |
| } | |
| if (res.status === 200) { | |
| const page = parseDjs(res.text); | |
| if (page) return { kind: "ok", page }; | |
| } | |
| return { kind: "challenge" }; | |
| } | |
| async function firstDjs(query: string, region: string, fresh: boolean): Promise<DjsOutcome> { | |
| const boot = await bootstrap(query, region, fresh); | |
| if (!boot?.djs) return { kind: "error" }; | |
| const outcome = await fetchDjs(boot.djs, boot.ua); | |
| if (outcome.kind === "ok") { | |
| // Remember which user agent opened this chain so later pages reuse it. | |
| return { kind: "ok", page: { ...outcome.page, ua: boot.ua } }; | |
| } | |
| // Do not reuse a session that DuckDuckGo has just refused. | |
| bootCache.delete(`${region}::${query}`); | |
| return outcome; | |
| } | |
| /** | |
| * DuckDuckGo answers quoted phrases and operators (`site:`, `-word`, …) with a | |
| * browser challenge when the request comes from a server. As a last resort the | |
| * same words are searched without the syntax — and the UI says so. | |
| * `"exact phrase" site:example.com -foo` -> `exact phrase example.com` | |
| */ | |
| export function relaxQuery(query: string): string | null { | |
| const relaxed = query | |
| .replace(/["“”„«»]/g, " ") | |
| .replace(/(^|\s)-\S+/g, "$1") | |
| .replace(/(^|\s)(?:site|inurl|intitle|intext|inbody|allintitle|allinurl|filetype|ext):/gi, "$1") | |
| .replace(/\s+/g, " ") | |
| .trim(); | |
| return relaxed && relaxed !== query.trim() ? relaxed : null; | |
| } | |
| /* ------------------------------ web results: html endpoint (page-1 fallback) */ | |
| function resolveHref(href: string): string { | |
| if (!href) return ""; | |
| if (href.startsWith("//duckduckgo.com/l/") || href.startsWith("/l/")) { | |
| try { | |
| const u = new URL(href.startsWith("//") ? `https:${href}` : `${DDG}${href}`); | |
| const target = u.searchParams.get("uddg"); | |
| if (target) return target; | |
| } catch { | |
| /* fall through */ | |
| } | |
| } | |
| if (href.startsWith("//")) return `https:${href}`; | |
| if (href.startsWith("/")) return `${DDG}${href}`; | |
| return href; | |
| } | |
| /** | |
| * The html endpoint only ever answers its first page from server IPs (its | |
| * "next" form is challenged), so it is kept purely as a safety net for page 1. | |
| */ | |
| async function htmlFirstPage(query: string, region: string, attempts = 2): Promise<WebResult[] | null> { | |
| for (let attempt = 0; attempt < attempts; attempt++) { | |
| const res = await ddgFetch( | |
| `${HTML_ENDPOINT}?q=${encodeURIComponent(query)}&kl=${region}`, | |
| attempt + 1, | |
| "text/html,application/xhtml+xml", | |
| "https://html.duckduckgo.com/", | |
| ); | |
| if (res && res.status === 200) { | |
| const $ = cheerio.load(res.text); | |
| const results: WebResult[] = []; | |
| const seen = new Set<string>(); | |
| $("div.result").each((_, el) => { | |
| const $el = $(el); | |
| if (($el.attr("class") ?? "").includes("result--ad")) return; | |
| const anchor = $el.find("a.result__a").first(); | |
| const url = resolveHref(anchor.attr("href") ?? ""); | |
| const title = clean(anchor.text()); | |
| if (!title || !isHttpUrl(url) || seen.has(url) || isDdgInternal(url)) return; | |
| seen.add(url); | |
| results.push({ | |
| kind: "web", | |
| title, | |
| url, | |
| snippet: clean($el.find(".result__snippet").first().text()), | |
| displayUrl: clean($el.find(".result__url").first().text()) || domainOf(url), | |
| date: null, | |
| }); | |
| }); | |
| if (results.length > 0 || $(".no-results").length > 0) return results; | |
| } | |
| if (attempt < attempts - 1) await sleep(500 + attempt * 500); | |
| } | |
| return null; | |
| } | |
| /** | |
| * DuckDuckGo serves about 200 web results per search: page 14 is refused by its | |
| * anomaly detection however slowly it is requested, and every refusal counts | |
| * against the server's IP. So the last page we ask for is 13. | |
| */ | |
| const WEB_MAX_PAGES = 13; | |
| /** | |
| * Last-resort safety net for the "All" tab. When DuckDuckGo refuses this server | |
| * outright (IP rate limit), show clearly labelled Wikipedia matches instead of | |
| * an empty page. Never cached, so a retry goes straight back to DuckDuckGo. | |
| */ | |
| async function wikipediaPage(query: string, cursor: Cursor | null): Promise<PageResult> { | |
| const page = cursor?.p ?? 1; | |
| const offset = cursor ? Number(cursor.n) : 0; | |
| if (!Number.isInteger(offset) || offset < 0 || offset > 9000) return ended(page); | |
| const lang = regionFor(query) === "xa-ar" ? "ar" : "en"; | |
| const api = | |
| `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&list=search&srlimit=20&sroffset=${offset}` + | |
| `&srprop=snippet&srsearch=${encodeURIComponent(query)}`; | |
| try { | |
| const res = await fetch(api, { | |
| headers: { "user-agent": "NexGO/1.0 (search demo)", accept: "application/json" }, | |
| cache: "no-store", | |
| signal: AbortSignal.timeout(8000), | |
| }); | |
| if (!res.ok) throw new Error(`wikipedia ${res.status}`); | |
| const data = (await res.json()) as { | |
| continue?: { sroffset?: number }; | |
| query?: { search?: Array<{ title?: string; snippet?: string }> }; | |
| }; | |
| const results: WebResult[] = []; | |
| for (const row of data.query?.search ?? []) { | |
| if (!row.title) continue; | |
| const slug = row.title.replace(/ /g, "_"); | |
| results.push({ | |
| kind: "web", | |
| title: `${row.title} - Wikipedia`, | |
| url: `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(slug)}`, | |
| snippet: clean(row.snippet), | |
| displayUrl: `${lang}.wikipedia.org/wiki/${slug}`, | |
| date: null, | |
| }); | |
| } | |
| const nextOffset = data.continue?.sroffset; | |
| return { | |
| results, | |
| related: [], | |
| page, | |
| next: | |
| typeof nextOffset === "number" && results.length > 0 | |
| ? encodeCursor({ k: "w", p: page + 1, u: 0, n: String(nextOffset) }) | |
| : null, | |
| degraded: false, | |
| fallback: results.length > 0 ? "wikipedia" : null, | |
| }; | |
| } catch { | |
| return failed(page, cursor); | |
| } | |
| } | |
| function buildContinuationQueries(baseQuery: string, related: RelatedItem[]): string[] { | |
| const seen = new Set<string>([baseQuery.trim().toLowerCase()]); | |
| const out: string[] = []; | |
| const push = (q: string) => { | |
| const cleanQ = q.replace(/\s+/g, " ").trim(); | |
| const key = cleanQ.toLowerCase(); | |
| if (!cleanQ || seen.has(key)) return; | |
| seen.add(key); | |
| out.push(cleanQ); | |
| }; | |
| for (const r of related) push(r.query); | |
| const isArabic = /[\u0600-\u06FF]/.test(baseQuery); | |
| const suffixes = isArabic | |
| ? ["شرح", "أفضل", "دليل", "مقارنة", "مراجعة", "أمثلة"] | |
| : ["guide", "overview", "tools", "best practices", "examples", "tutorial", "comparison", "2026"]; | |
| for (const s of suffixes) { | |
| if (out.length >= 10) break; | |
| push(`${baseQuery} ${s}`); | |
| } | |
| return out.slice(0, 10); | |
| } | |
| async function continueViaSubQueries( | |
| pageNumber: number, | |
| queue: string[], | |
| uaHint: number, | |
| ): Promise<PageResult> { | |
| const remaining = queue.slice(); | |
| while (remaining.length > 0) { | |
| const subQuery = remaining.shift()!; | |
| const reg = regionFor(subQuery); | |
| const outcome = await firstDjs(subQuery, reg, true); | |
| if (outcome.kind === "ok" && outcome.page.results.length > 0) { | |
| const nextQ = remaining[0]; | |
| return { | |
| results: outcome.page.results, | |
| related: [], | |
| page: pageNumber, | |
| next: | |
| nextQ && pageNumber < WEB_MAX_PAGES | |
| ? encodeCursor({ k: "q", p: pageNumber + 1, u: (uaHint + 1) % USER_AGENTS.length, n: nextQ, alt: remaining.slice(1) }) | |
| : null, | |
| degraded: false, | |
| fallback: null, | |
| }; | |
| } | |
| if (outcome.kind === "blocked") break; | |
| } | |
| return ended(pageNumber); | |
| } | |
| async function webPage(query: string, cursor: Cursor | null): Promise<PageResult> { | |
| const region = regionFor(query); | |
| if (cursor) { | |
| if (cursor.k === "w") return wikipediaPage(query, cursor); | |
| if (cursor.k === "q") { | |
| const queue = [cursor.n, ...(cursor.alt ?? [])]; | |
| const sub = await continueViaSubQueries(cursor.p, queue, cursor.u); | |
| if (sub.results.length > 0) return sub; | |
| const wiki = await wikipediaPage(query, { k: "w", p: cursor.p, u: 0, n: String((cursor.p - 1) * 20) }); | |
| return wiki.results.length > 0 ? wiki : ended(cursor.p); | |
| } | |
| const url = cursor.k === "d" ? pinnedUrl(cursor.n, LINKS, "/d.js") : null; | |
| if (!url) return ended(cursor.p); | |
| for (let attempt = 0; attempt < 2; attempt++) { | |
| const outcome = await fetchDjs(url, (cursor.u + attempt) % USER_AGENTS.length); | |
| if (outcome.kind === "ok" && outcome.page.results.length > 0) { | |
| const { page } = outcome; | |
| const alt = cursor.alt ?? buildContinuationQueries(query, []); | |
| return { | |
| results: page.results, | |
| related: [], | |
| page: cursor.p, | |
| next: | |
| page.next && cursor.p < WEB_MAX_PAGES | |
| ? encodeCursor({ k: "d", p: cursor.p + 1, u: cursor.u, n: page.next, alt }) | |
| : alt.length > 0 && cursor.p < WEB_MAX_PAGES | |
| ? encodeCursor({ k: "q", p: cursor.p + 1, u: cursor.u, n: alt[0], alt: alt.slice(1) }) | |
| : null, | |
| degraded: false, | |
| fallback: null, | |
| }; | |
| } | |
| if (outcome.kind !== "error") break; | |
| await sleep(400); | |
| } | |
| // DuckDuckGo refused s=10+ (HTTP 202 challenge/block): seamlessly continue via | |
| // related/expanded DuckDuckGo p1 queries (which succeed 200 OK), then Wikipedia. | |
| const queue = cursor.alt && cursor.alt.length > 0 ? cursor.alt : buildContinuationQueries(query, []); | |
| const sub = await continueViaSubQueries(cursor.p, queue, cursor.u); | |
| if (sub.results.length > 0) return sub; | |
| const wiki = await wikipediaPage(query, { k: "w", p: cursor.p, u: 0, n: "0" }); | |
| return wiki.results.length > 0 ? wiki : ended(cursor.p); | |
| } | |
| const firstPage = (page: DjsPage, extra: Partial<PageResult> = {}): PageResult => { | |
| const alt = buildContinuationQueries(query, page.related); | |
| return { | |
| results: page.results, | |
| related: page.related, | |
| page: 1, | |
| next: page.next | |
| ? encodeCursor({ k: "d", p: 2, u: page.ua ?? 0, n: page.next, alt }) | |
| : alt.length > 0 | |
| ? encodeCursor({ k: "q", p: 2, u: page.ua ?? 0, n: alt[0], alt: alt.slice(1) }) | |
| : null, | |
| degraded: false, | |
| fallback: null, | |
| ...extra, | |
| }; | |
| }; | |
| // 1. The exact query through d.js (one retry, with a new session, on network errors only). | |
| let outcome = await firstDjs(query, region, false); | |
| if (outcome.kind === "error") { | |
| await sleep(350); | |
| outcome = await firstDjs(query, region, true); | |
| } | |
| if (outcome.kind === "ok") return firstPage(outcome.page); | |
| // 2. The exact query through the html endpoint (understands operators, first page only). | |
| const html = await htmlFirstPage(query, region, outcome.kind === "error" ? 2 : 1); | |
| if (html) { | |
| return { results: html, related: [], page: 1, next: null, degraded: false, fallback: html.length > 0 ? "html" : null }; | |
| } | |
| // 3. Quoted / operator query that DuckDuckGo challenged: search the plain words instead. | |
| const relaxed = outcome.kind === "challenge" ? relaxQuery(query) : null; | |
| if (relaxed) { | |
| const plain = await firstDjs(relaxed, regionFor(relaxed), false); | |
| if (plain.kind === "ok" && plain.page.results.length > 0) { | |
| return firstPage(plain.page, { fallback: "relaxed", shownFor: relaxed }); | |
| } | |
| } | |
| // 4. DuckDuckGo is refusing this server altogether: keep the page useful. | |
| const wiki = await wikipediaPage(query, null); | |
| if (wiki.results.length > 0) return wiki; | |
| const plainWords = relaxQuery(query); | |
| if (plainWords) { | |
| const wikiPlain = await wikipediaPage(plainWords, null); | |
| if (wikiPlain.results.length > 0) return wikiPlain; | |
| } | |
| return failed(1, null); | |
| } | |
| /* -------------------------------------- JSON verticals: news / videos / photos */ | |
| const ENDPOINT: Record<Vertical, string> = { news: "news.js", videos: "v.js", images: "i.js" }; | |
| function firstVerticalUrl(tab: Vertical, query: string, region: string, vqd: string): string { | |
| const q = encodeURIComponent(query); | |
| if (tab === "news") return `${DDG}/news.js?l=${region}&o=json&noamp=1&q=${q}&vqd=${vqd}&p=-1`; | |
| if (tab === "videos") return `${DDG}/v.js?l=${region}&o=json&q=${q}&vqd=${vqd}&f=,,,&p=-1`; | |
| return `${DDG}/i.js?l=${region}&o=json&q=${q}&vqd=${vqd}&f=,,,&p=1`; | |
| } | |
| function withVqd(url: string, vqd: string, replace = false): string { | |
| if (/[?&]vqd=/.test(url)) return replace ? url.replace(/([?&]vqd=)[^&]*/, `$1${vqd}`) : url; | |
| return `${url}${url.includes("?") ? "&" : "?"}vqd=${vqd}`; | |
| } | |
| function mapNews(rows: Json[]): NewsResult[] { | |
| const out: NewsResult[] = []; | |
| const seen = new Set<string>(); | |
| for (const row of rows) { | |
| const url = str(row.url); | |
| const title = clean(row.title); | |
| if (!title || !isHttpUrl(url) || seen.has(url)) continue; | |
| seen.add(url); | |
| out.push({ | |
| kind: "news", | |
| title, | |
| url, | |
| snippet: clean(row.excerpt) || clean(row.description), | |
| source: clean(row.source) || domainOf(url), | |
| date: toIso(row.date), | |
| image: httpOrNull(row.image), | |
| }); | |
| } | |
| return out; | |
| } | |
| function mapVideos(rows: Json[]): VideoResult[] { | |
| const out: VideoResult[] = []; | |
| const seen = new Set<string>(); | |
| for (const row of rows) { | |
| const url = str(row.content) || str(row.url); | |
| const title = clean(row.title); | |
| if (!title || !isHttpUrl(url) || seen.has(url)) continue; | |
| seen.add(url); | |
| const images = (row.images && typeof row.images === "object" ? row.images : {}) as Json; | |
| const stats = (row.statistics && typeof row.statistics === "object" ? row.statistics : {}) as Json; | |
| out.push({ | |
| kind: "video", | |
| title, | |
| url, | |
| snippet: clean(row.description).slice(0, 320), | |
| // DuckDuckGo nests thumbnails under `images` and view counts under `statistics`. | |
| thumbnail: | |
| httpOrNull(images.large) ?? httpOrNull(images.medium) ?? httpOrNull(images.small) ?? httpOrNull(row.image), | |
| duration: str(row.duration) || null, | |
| views: | |
| typeof stats.viewCount === "number" | |
| ? stats.viewCount | |
| : typeof row.views === "number" | |
| ? row.views | |
| : null, | |
| published: toIso(row.published), | |
| source: clean(row.uploader) || clean(row.publisher) || domainOf(url), | |
| }); | |
| } | |
| return out; | |
| } | |
| function toDimension(value: unknown): number | null { | |
| const n = typeof value === "number" ? value : typeof value === "string" ? parseInt(value, 10) : NaN; | |
| return Number.isFinite(n) && n > 0 ? n : null; | |
| } | |
| function mapImages(rows: Json[]): ImageResult[] { | |
| const out: ImageResult[] = []; | |
| const seen = new Set<string>(); | |
| for (const row of rows) { | |
| const image = str(row.image); | |
| if (!isHttpUrl(image) || seen.has(image)) continue; | |
| seen.add(image); | |
| const pageUrl = httpOrNull(row.url) ?? image; | |
| out.push({ | |
| kind: "image", | |
| title: clean(row.title) || domainOf(pageUrl), | |
| pageUrl, | |
| image, | |
| thumbnail: httpOrNull(row.thumbnail) ?? image, | |
| width: toDimension(row.width), | |
| height: toDimension(row.height), | |
| source: domainOf(pageUrl), | |
| }); | |
| } | |
| return out; | |
| } | |
| interface VerticalPage { | |
| results: SearchResult[]; | |
| next: string | null; | |
| } | |
| type VerticalOutcome = | |
| | { kind: "ok"; page: VerticalPage } | |
| /** DuckDuckGo answered with its anti-bot challenge: retrying right away is pointless. */ | |
| | { kind: "blocked" } | |
| /** Network failure, server error or an unreadable payload (e.g. an expired token). */ | |
| | { kind: "error" }; | |
| async function fetchVertical(tab: Vertical, url: string, ua: number): Promise<VerticalOutcome> { | |
| const res = await ddgFetch(url, ua, "application/json, text/javascript, */*; q=0.01"); | |
| if (!res || res.status >= 500) return { kind: "error" }; | |
| let data: { challenge?: unknown; results?: unknown; next?: unknown } | null = null; | |
| try { | |
| data = JSON.parse(res.text); | |
| } catch { | |
| /* not JSON: handled below */ | |
| } | |
| const isObject = data !== null && typeof data === "object"; | |
| if (res.status === 403 || res.status === 429 || (isObject && data?.challenge)) return { kind: "blocked" }; | |
| if (res.status !== 200 || !isObject || !Array.isArray(data?.results)) return { kind: "error" }; | |
| const rows = data.results as Json[]; | |
| const results = tab === "news" ? mapNews(rows) : tab === "videos" ? mapVideos(rows) : mapImages(rows); | |
| return { kind: "ok", page: { results, next: typeof data.next === "string" && data.next ? data.next : null } }; | |
| } | |
| /** Openly licensed photos, used only when DuckDuckGo challenges the image endpoint. */ | |
| async function commonsPage(query: string, cursor: Cursor | null): Promise<PageResult> { | |
| const page = cursor?.p ?? 1; | |
| const offset = cursor ? Number(cursor.n) : 0; | |
| if (!Number.isInteger(offset) || offset < 0 || offset > 10_000) return ended(page); | |
| const api = | |
| "https://commons.wikimedia.org/w/api.php?action=query&format=json&generator=search" + | |
| `&gsrsearch=${encodeURIComponent(`${query} filetype:bitmap`)}&gsrnamespace=6&gsrlimit=50&gsroffset=${offset}` + | |
| "&prop=imageinfo&iiprop=url%7Csize%7Cmime&iiurlwidth=500"; | |
| try { | |
| const res = await fetch(api, { | |
| headers: { "user-agent": "NexGO/1.0 (search demo)", accept: "application/json" }, | |
| cache: "no-store", | |
| signal: AbortSignal.timeout(10_000), | |
| }); | |
| if (!res.ok) throw new Error(`commons ${res.status}`); | |
| const data = (await res.json()) as { | |
| continue?: { gsroffset?: number }; | |
| query?: { | |
| pages?: Record< | |
| string, | |
| { | |
| title?: string; | |
| index?: number; | |
| imageinfo?: Array<{ | |
| url?: string; | |
| thumburl?: string; | |
| descriptionurl?: string; | |
| width?: number; | |
| height?: number; | |
| mime?: string; | |
| }>; | |
| } | |
| >; | |
| }; | |
| }; | |
| const rows = Object.values(data.query?.pages ?? {}).sort((a, b) => (a.index ?? 0) - (b.index ?? 0)); | |
| const results: ImageResult[] = []; | |
| for (const row of rows) { | |
| const info = row.imageinfo?.[0]; | |
| if (!info?.url || (info.mime && !info.mime.startsWith("image/"))) continue; | |
| const name = row.title ?? ""; | |
| results.push({ | |
| kind: "image", | |
| title: name.replace(/^File:/, "").replace(/\.[a-z0-9]+$/i, "") || "Wikimedia Commons image", | |
| pageUrl: info.descriptionurl || `https://commons.wikimedia.org/wiki/${encodeURIComponent(name)}`, | |
| image: info.url, | |
| thumbnail: info.thumburl || info.url, | |
| width: toDimension(info.width), | |
| height: toDimension(info.height), | |
| source: "commons.wikimedia.org", | |
| }); | |
| } | |
| const nextOffset = data.continue?.gsroffset; | |
| return { | |
| results, | |
| related: [], | |
| page, | |
| next: | |
| typeof nextOffset === "number" && results.length > 0 | |
| ? encodeCursor({ k: "c", p: page + 1, u: 0, n: String(nextOffset) }) | |
| : null, | |
| degraded: false, | |
| fallback: results.length > 0 ? "commons" : null, | |
| }; | |
| } catch { | |
| return failed(page, cursor); | |
| } | |
| } | |
| async function verticalPage(tab: Vertical, query: string, cursor: Cursor | null): Promise<PageResult> { | |
| const region = regionFor(query); | |
| if (cursor) { | |
| if (cursor.k === "c") return tab === "images" ? commonsPage(query, cursor) : ended(cursor.p); | |
| const base = cursor.k === "j" ? pinnedUrl(cursor.n, DDG, `/${ENDPOINT[tab]}`) : null; | |
| if (!base || !cursor.v) return ended(cursor.p); | |
| let vqd = cursor.v; | |
| let url = withVqd(base, vqd); | |
| for (let attempt = 0; attempt < 3; attempt++) { | |
| const outcome = await fetchVertical(tab, url, cursor.u); | |
| if (outcome.kind === "ok") { | |
| const { page } = outcome; | |
| return { | |
| results: page.results, | |
| related: [], | |
| page: cursor.p, | |
| next: page.next ? encodeCursor({ k: "j", p: cursor.p + 1, u: cursor.u, n: page.next, v: vqd }) : null, | |
| degraded: false, | |
| fallback: null, | |
| }; | |
| } | |
| // Challenged: report it at once; the visitor can retry this same page later. | |
| if (outcome.kind === "blocked") break; | |
| if (attempt === 1) { | |
| // The token may simply have expired on a long scroll session: mint a new one. | |
| const boot = await bootstrap(query, region, true); | |
| if (boot) { | |
| vqd = boot.vqd; | |
| url = withVqd(base, vqd, true); | |
| } | |
| } | |
| await sleep(500 + attempt * 600); | |
| } | |
| if (tab === "images") return commonsPage(query, { k: "c", p: cursor.p, u: 0, n: String((cursor.p - 1) * 50) }); | |
| return ended(cursor.p); | |
| } | |
| for (let attempt = 0; attempt < 3; attempt++) { | |
| const boot = await bootstrap(query, region, attempt > 0); | |
| if (boot) { | |
| const outcome = await fetchVertical(tab, firstVerticalUrl(tab, query, region, boot.vqd), boot.ua); | |
| if (outcome.kind === "ok") { | |
| const { page } = outcome; | |
| return { | |
| results: page.results, | |
| related: [], | |
| page: 1, | |
| next: page.next ? encodeCursor({ k: "j", p: 2, u: boot.ua, n: page.next, v: boot.vqd }) : null, | |
| degraded: false, | |
| fallback: null, | |
| }; | |
| } | |
| if (outcome.kind === "blocked") break; | |
| } | |
| await sleep(450 + attempt * 500); | |
| } | |
| if (tab === "images") return commonsPage(query, null); | |
| return ended(1); | |
| } | |
| /* ------------------------------------------------------------------- public */ | |
| /** Fetches exactly one DuckDuckGo page for the tab, untruncated. */ | |
| export function searchPage(tab: SearchTab, query: string, cursor: Cursor | null): Promise<PageResult> { | |
| return tab === "all" ? webPage(query, cursor) : verticalPage(tab, query, cursor); | |
| } | |
| export async function instantAnswer(query: string): Promise<InstantAnswer | null> { | |
| try { | |
| const res = await fetch( | |
| `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1&no_redirect=1&skip_disambig=1&t=nexgo`, | |
| { | |
| headers: { "user-agent": USER_AGENTS[0], accept: "application/json" }, | |
| cache: "no-store", | |
| signal: AbortSignal.timeout(4000), | |
| }, | |
| ); | |
| if (!res.ok) return null; | |
| const data = (await res.json()) as Json; | |
| const text = clean(data.AbstractText) || clean(data.Answer) || clean(data.Definition); | |
| if (!text) return null; | |
| // The API returns site-relative image paths such as "/i/abc123.png". | |
| const rawImage = str(data.Image); | |
| const image = rawImage ? (rawImage.startsWith("/") ? `${DDG}${rawImage}` : httpOrNull(rawImage)) : null; | |
| return { | |
| heading: str(data.Heading) || null, | |
| text, | |
| url: httpOrNull(data.AbstractURL) ?? httpOrNull(data.DefinitionURL), | |
| image, | |
| label: str(data.AbstractSource) || str(data.DefinitionSource) || "Instant answer", | |
| }; | |
| } catch { | |
| return null; | |
| } | |
| } | |