src/search.js

// ----------------------------------------------------------------------------
// Web search provider — Tavily (https://tavily.com).
//
// WHY THIS CHANGED: the old version scraped DuckDuckGo directly. DuckDuckGo
// blocks datacenter IPs (Hugging Face, Render, Fly, etc.), so in production it
// returned ZERO results ({"sources":[],"notes":""}). Tavily is a search API
// built for LLMs: it returns clean, relevant content snippets per result, so we
// no longer scrape or fetch pages ourselves — it's faster and reliable.
//
// SETUP (one-time):
//   1. Get a free API key at https://app.tavily.com  (1,000 searches/month,
//      no credit card). It looks like:  tvly-xxxxxxxxxxxxxxxx
//   2. Add it to your Hugging Face Space as a SECRET named TAVILY_API_KEY
//      (Space → Settings → Variables and secrets → New secret).
//
// The exported interface (webSearch, buildSourceNotes, domainOf) is unchanged,
// so server.js keeps working without edits.
// ----------------------------------------------------------------------------

const TAVILY_API_KEY = process.env.TAVILY_API_KEY || '';
const TAVILY_URL = 'https://api.tavily.com/search';

async function fetchWithTimeout(url, init = {}, timeoutMs = 15000) {
  const controller = new AbortController();
  const id = setTimeout(() => controller.abort(), timeoutMs);
  try {
    return await fetch(url, { ...init, signal: controller.signal });
  } finally {
    clearTimeout(id);
  }
}

export function domainOf(url) {
  return String(url).replace(/^https?:\/\//, '').split('/')[0];
}

// Query Tavily -> [{ title, url, snippet }]
export async function webSearch(query, maxResults = 6) {
  const q = String(query || '').trim();
  if (!q) return [];
  if (!TAVILY_API_KEY) {
    console.warn('[search] TAVILY_API_KEY is not set — web search disabled.');
    return [];
  }

  try {
    const res = await fetchWithTimeout(TAVILY_URL, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${TAVILY_API_KEY}`,
      },
      body: JSON.stringify({
        query: q,
        max_results: Math.min(Math.max(maxResults, 1), 10),
        search_depth: 'basic',
        include_answer: false,
        include_raw_content: false,
      }),
    });

    if (!res.ok) {
      const text = await res.text().catch(() => '');
      throw new Error(`tavily HTTP ${res.status} ${text.slice(0, 200)}`);
    }

    const data = await res.json();
    const results = Array.isArray(data.results) ? data.results : [];
    return results.map((r) => ({
      title: r.title || domainOf(r.url || ''),
      url: r.url || '',
      snippet: String(r.content || '').trim(),
    }));
  } catch (e) {
    console.warn('[search] tavily failed:', e.message);
    return [];
  }
}

// Turn results into numbered "source notes" the on-device model cites as [1],[2].
// Tavily already returns the relevant content per result, so there is nothing to
// fetch — we just format. (Extra args kept for signature compatibility.)
export async function buildSourceNotes(
  query,
  results,
  _fetchTopK = 4,
  _maxFactsPerSource = 4,
  maxTotalChars = 9000,
) {
  const blocks = (results || []).map((r, i) => {
    const facts = String(r.snippet || '').replace(/\s+/g, ' ').trim();
    return (
      `[${i + 1}] Title: ${r.title}\nURL: ${r.url}\nExtracted facts:\n` +
      `- ${facts || '(no snippet returned)'}`
    );
  });
  return blocks.join('\n\n').slice(0, maxTotalChars);
}