web search: use Tavily instead of DuckDuckGo

src/search.js

@@ -1,239 +1,98 @@
 // ----------------------------------------------------------------------------
-// Web search provider (server-side) — DuckDuckGo ONLY, no API keys.
+// Web search provider — Tavily (https://tavily.com).
 //
-// This is the Node port of the Python reference (duckduckgo_search + fact
-// extraction):
-//   1. Query DuckDuckGo:  POST https://html.duckduckgo.com/html/
-//      (fallback:         POST https://lite.duckduckgo.com/lite/)
-//   2. Parse result links/snippets TOLERANTLY. This is the bug fix: the old
-//      regex required class="result-link" to appear BEFORE href, with double
-//      quotes. DDG's real markup is  <a rel="nofollow" href="..."
-//      class='result-link'>  (href first, SINGLE quotes), so the old parser
-//      matched 0 results every time and the app said search "failed".
-//   3. Fetch the top pages, split them into sentences, and keep only the
-//      sentences that score well against the query keywords — the model
-//      receives facts, not huge noisy pages (same scoring as the Python code).
-//   4. Emit numbered source-note blocks the model can cite as [1], [2], ...
+// WHY THIS CHANGED: the old version scraped DuckDuckGo directly. DuckDuckGo
+// blocks datacenter IPs (Hugging Face, Render, Fly, etc.), so in production it
+// returned ZERO results ({"sources":[],"notes":""}). Tavily is a search API
+// built for LLMs: it returns clean, relevant content snippets per result, so we
+// no longer scrape or fetch pages ourselves — it's faster and reliable.
+//
+// SETUP (one-time):
+//   1. Get a free API key at https://app.tavily.com  (1,000 searches/month,
+//      no credit card). It looks like:  tvly-xxxxxxxxxxxxxxxx
+//   2. Add it to your Hugging Face Space as a SECRET named TAVILY_API_KEY
+//      (Space → Settings → Variables and secrets → New secret).
+//
+// The exported interface (webSearch, buildSourceNotes, domainOf) is unchanged,
+// so server.js keeps working without edits.
 // ----------------------------------------------------------------------------
 
-import { convert as htmlToText } from 'html-to-text';
-
-const UA =
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
+const TAVILY_API_KEY = process.env.TAVILY_API_KEY || '';
+const TAVILY_URL = 'https://api.tavily.com/search';
 
-async function fetchWithTimeout(url, init = {}, timeoutMs = 10000) {
+async function fetchWithTimeout(url, init = {}, timeoutMs = 15000) {
   const controller = new AbortController();
   const id = setTimeout(() => controller.abort(), timeoutMs);
   try {
-    return await fetch(url, { redirect: 'follow', ...init, signal: controller.signal });
+    return await fetch(url, { ...init, signal: controller.signal });
   } finally {
     clearTimeout(id);
   }
 }
 
-// ---- DuckDuckGo search ------------------------------------------------------
+export function domainOf(url) {
+  return String(url).replace(/^https?:\/\//, '').split('/')[0];
+}
 
+// Query Tavily -> [{ title, url, snippet }]
 export async function webSearch(query, maxResults = 6) {
   const q = String(query || '').trim();
   if (!q) return [];
-
-  let results = [];
-  try {
-    results = await ddgSearch('https://html.duckduckgo.com/html/', q);
-  } catch (e) {
-    console.warn('[search] html endpoint failed:', e.message);
-  }
-  if (results.length === 0) {
-    try {
-      results = await ddgSearch('https://lite.duckduckgo.com/lite/', q);
-    } catch (e) {
-      console.warn('[search] lite endpoint failed:', e.message);
-    }
-  }
-  return results.slice(0, maxResults);
-}
-
-async function ddgSearch(endpoint, query) {
-  const res = await fetchWithTimeout(endpoint, {
-    method: 'POST',
-    headers: {
-      'User-Agent': UA,
-      'Content-Type': 'application/x-www-form-urlencoded',
-      Accept: 'text/html,application/xhtml+xml',
-      'Accept-Language': 'en-US,en;q=0.9',
-    },
-    body: `q=${encodeURIComponent(query)}&kl=wt-wt`,
-  });
-  if (!res.ok) throw new Error(`ddg HTTP ${res.status}`);
-  return parseDdgHtml(await res.text());
-}
-
-// Tolerant parser for BOTH DDG layouts (exported so it can be unit-tested):
-//   html.duckduckgo.com -> <a class="result__a" href="...">   + class="result__snippet"
-//   lite.duckduckgo.com -> <a href="..." class='result-link'> + class='result-snippet'
-// Attribute order and quote style are NOT assumed.
-export function parseDdgHtml(html) {
-  const snippets = [];
-  const snipRe =
-    /class=["'][^"']*(?:result__snippet|result-snippet)[^"']*["'][^>]*>([\s\S]*?)<\/(?:a|td|div|span)>/gi;
-  let sm;
-  while ((sm = snipRe.exec(html))) snippets.push(clean(sm[1]));
-
-  const results = [];
-  const seen = new Set();
-  const aRe = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
-  let m;
-  while ((m = aRe.exec(html))) {
-    const attrs = m[1];
-    if (!/class=["'][^"']*(?:result__a|result-link)[^"']*["']/i.test(attrs)) continue;
-    const hrefMatch = attrs.match(/href=["']([^"']+)["']/i);
-    if (!hrefMatch) continue;
-    if (/duckduckgo\.com\/y\.js|ad_provider=|ad_domain=/i.test(hrefMatch[1])) continue; // skip ads
-    const url = decodeDdgUrl(hrefMatch[1]);
-    if (!url || seen.has(url)) continue;
-    seen.add(url);
-    results.push({ title: clean(m[2]), url, snippet: snippets[results.length] ?? '' });
+  if (!TAVILY_API_KEY) {
+    console.warn('[search] TAVILY_API_KEY is not set — web search disabled.');
+    return [];
   }
-  return results;
-}
-
-function decodeDdgUrl(href) {
-  // result links point at //duckduckgo.com/l/?uddg=<real-url-encoded>&rut=...
-  const m = href.match(/[?&]uddg=([^&]+)/);
-  if (m) {
-    try {
-      return decodeURIComponent(m[1]);
-    } catch {
-      /* fall through */
-    }
-  }
-  if (href.startsWith('//')) return 'https:' + href;
-  return href.startsWith('http') ? href : '';
-}
-
-function clean(s) {
-  return String(s)
-    .replace(/<[^>]+>/g, '')
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&#x27;/g, "'")
-    .replace(/&#39;/g, "'")
-    .replace(/&quot;/g, '"')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&amp;/g, '&')
-    .replace(/\s+/g, ' ')
-    .trim();
-}
-
-export function domainOf(url) {
-  return String(url).replace(/^https?:\/\//, '').split('/')[0];
-}
-
-// ---- fact extraction (port of the Python logic) -----------------------------
 
-const STOP_WORDS = new Set([
-  'the', 'a', 'an', 'and', 'or', 'to', 'of', 'in', 'on', 'for', 'with', 'about',
-  'give', 'me', 'tell', 'current', 'latest', 'recent', 'what', 'who', 'is', 'are',
-  'their', 'its', 'this', 'that', 'from', 'by', 'as', 'at', 'be', 'was', 'were',
-]);
-
-function simpleWords(text) {
-  const words = String(text).toLowerCase().match(/[a-z0-9][a-z0-9\-._]*/g) ?? [];
-  return words.filter((w) => !STOP_WORDS.has(w) && w.length > 2);
-}
-
-function splitSentences(text) {
-  return String(text)
-    .replace(/\s+/g, ' ')
-    .trim()
-    .split(/(?<=[.!?])\s+/)
-    .map((p) => p.trim())
-    .filter((p) => p.length >= 40 && p.length <= 400);
-}
-
-async function fetchPageText(url, maxChars = 5000) {
   try {
-    const res = await fetchWithTimeout(url, { headers: { 'User-Agent': UA } }, 8000);
-    const ct = (res.headers.get('content-type') || '').toLowerCase();
-    if (!ct.includes('text/html') && !ct.includes('application/xhtml')) return '';
-    const text = htmlToText(await res.text(), {
-      wordwrap: false,
-      selectors: [
-        { selector: 'a', options: { ignoreHref: true } },
-        { selector: 'img', format: 'skip' },
-        { selector: 'nav', format: 'skip' },
-        { selector: 'footer', format: 'skip' },
-        { selector: 'header', format: 'skip' },
-        { selector: 'form', format: 'skip' },
-      ],
+    const res = await fetchWithTimeout(TAVILY_URL, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${TAVILY_API_KEY}`,
+      },
+      body: JSON.stringify({
+        query: q,
+        max_results: Math.min(Math.max(maxResults, 1), 10),
+        search_depth: 'basic',
+        include_answer: false,
+        include_raw_content: false,
+      }),
     });
-    return text.replace(/\s+/g, ' ').trim().slice(0, maxChars);
-  } catch {
-    return '';
-  }
-}
-
-// Keep only the sentences most related to the query (same scoring as Python).
-function extractRelevantFacts(query, title, snippet, pageText, maxSentences = 4) {
-  const keywords = new Set(simpleWords(query));
-  const titleWords = new Set(simpleWords(title));
-
-  const candidates = [];
-  if (snippet) candidates.push(snippet);
-  candidates.push(...splitSentences(pageText));
 
-  const scored = [];
-  for (const sent of candidates) {
-    const lower = sent.toLowerCase();
-    const words = new Set(simpleWords(sent));
-    let score = 0;
-    for (const w of words) if (keywords.has(w)) score += 3; // keyword overlap
-    for (const w of words) if (titleWords.has(w)) score += 1; // title overlap
-    if (/\b20\d{2}\b/.test(sent)) score += 2; // dates
-    if (/\bv?\d+\.\d+(\.\d+)?\b/.test(lower)) score += 3; // versions
-    if (/\$|billion|million|net worth|released|version|latest|rank|source of wealth/.test(lower))
-      score += 2;
-    if (/reuters|bbc|al jazeera|forbes|pypi|github|hugging face|official/.test(lower)) score += 1;
-    if (score > 0) scored.push({ score, sent });
-  }
-  scored.sort((a, b) => b.score - a.score);
-
-  const selected = [];
-  const seen = new Set();
-  for (const { sent } of scored) {
-    const key = sent.slice(0, 120).toLowerCase();
-    if (!seen.has(key)) {
-      selected.push(sent);
-      seen.add(key);
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      throw new Error(`tavily HTTP ${res.status} ${text.slice(0, 200)}`);
     }
-    if (selected.length >= maxSentences) break;
+
+    const data = await res.json();
+    const results = Array.isArray(data.results) ? data.results : [];
+    return results.map((r) => ({
+      title: r.title || domainOf(r.url || ''),
+      url: r.url || '',
+      snippet: String(r.content || '').trim(),
+    }));
+  } catch (e) {
+    console.warn('[search] tavily failed:', e.message);
+    return [];
   }
-  return selected;
 }
 
-// Fetch the top pages and turn everything into compact numbered "source notes"
-// for the on-device model. NOTE: now takes the query first (needed for scoring).
+// Turn results into numbered "source notes" the on-device model cites as [1],[2].
+// Tavily already returns the relevant content per result, so there is nothing to
+// fetch — we just format. (Extra args kept for signature compatibility.)
 export async function buildSourceNotes(
   query,
   results,
-  fetchTopK = 4,
-  maxFactsPerSource = 4,
+  _fetchTopK = 4,
+  _maxFactsPerSource = 4,
   maxTotalChars = 9000,
 ) {
-  const blocks = [];
-  for (let i = 0; i < results.length; i++) {
-    const r = results[i];
-    let pageText = '';
-    if (i < fetchTopK && String(r.url).startsWith('http')) {
-      pageText = await fetchPageText(r.url);
-    }
-    let facts = extractRelevantFacts(query, r.title, r.snippet, pageText, maxFactsPerSource);
-    if (facts.length === 0 && r.snippet) facts = [r.snippet];
-
-    blocks.push(
+  const blocks = (results || []).map((r, i) => {
+    const facts = String(r.snippet || '').replace(/\s+/g, ' ').trim();
+    return (
       `[${i + 1}] Title: ${r.title}\nURL: ${r.url}\nExtracted facts:\n` +
-        facts.map((f) => `- ${f}`).join('\n'),
+      `- ${facts || '(no snippet returned)'}`
     );
-  }
+  });
   return blocks.join('\n\n').slice(0, maxTotalChars);
-}
+}