Runtime/AgentCore/Tools/Builtin/WebSearchTool.cs

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Newtonsoft.Json.Linq;

namespace OnDeviceAgent.AgentCore
{
public sealed class WebSearchTool : IAgentTool
{
    const string DdgApiUrl = "https://api.duckduckgo.com/";
    const string DdgHtmlUrl = "https://html.duckduckgo.com/html/";
    const int MaxSummaryChars = 600;

    static readonly HttpClient s_Http = CreateClient();

    static HttpClient CreateClient()
    {
        var c = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
        c.DefaultRequestHeaders.Add("User-Agent",
            "Mozilla/5.0 (compatible; OnDeviceAgent/1.0; on-device voice agent)");
        return c;
    }

    public string Name => "WebSearch";
    public string Description =>
        "Search the web for current events or topics outside the indexed knowledge base (e.g. 'search the web for...', " +
        "'최신 뉴스 검색해줘', '인터넷에 찾아봐'), or as a fallback when SearchKnowledge returned no relevant passage. " +
        "For questions about the documented subject, try SearchKnowledge FIRST; if it finds nothing, call WebSearch " +
        "with a concise keyword query, not a full sentence. If the results do not answer the question, call WebSearch " +
        "again with simpler or different keywords before giving up.";

    public AITool ToAIFunction(AgentToolContext context)
    {
        return AIFunctionFactory.Create(
            new Func<string, Task<string>>(
                async (query) => await InvokeAsync(context, query).ConfigureAwait(false)),
            name: Name,
            description: Description);
    }

    async Task<string> InvokeAsync(AgentToolContext ctx, string query)
    {
        var argsJson = "{\"query\":\"" + (query ?? string.Empty).Replace("\"", "'") + "\"}";
        if (!ctx.Tracker.TryBegin(Name, "Tool WebSearch called.", argsJson, out var blocked))
            return blocked;

        if (!await ctx.Approval.IsApprovedOrRequest(this, ctx).ConfigureAwait(false))
            return ctx.Tracker.Finish(Name, argsJson, BuildApprovalRequiredMessage());

        if (string.IsNullOrWhiteSpace(query))
            return ctx.Tracker.Finish(Name, argsJson, "WebSearch needs a search query in 'query'.");

        try
        {
            var summary = await SearchAsync(query, ctx.CancellationToken).ConfigureAwait(false);
            return ctx.Tracker.Finish(Name, argsJson, summary);
        }
        catch (OperationCanceledException)
        {
            throw; // honour run cancellation
        }
        catch (Exception ex)
        {
            return ctx.Tracker.Finish(Name, argsJson, "WebSearch failed: " + ex.Message);
        }
    }

    static async Task<string> SearchAsync(string rawQuery, CancellationToken ct)
    {
        var query = CleanQuery(rawQuery);

        var result = await RunSourcesAsync(query, ct).ConfigureAwait(false);
        if (!string.IsNullOrEmpty(result))
            return result;

        // An over-specific query often matches nothing; retry once with just the core keywords.
        var simpler = Simplify(query);
        if (simpler != null)
        {
            result = await RunSourcesAsync(simpler, ct).ConfigureAwait(false);
            if (!string.IsNullOrEmpty(result))
                return result;
        }

        return "No concise result found for \"" + query + "\". Call WebSearch again with simpler or different keywords, or tell the user you couldn't find it.";
    }

    static async Task<string> RunSourcesAsync(string query, CancellationToken ct)
    {
        var ddg = await TryDuckDuckGoInstantAsync(query, ct).ConfigureAwait(false);
        if (!string.IsNullOrEmpty(ddg))
            return ddg;

        var web = await TryDuckDuckGoHtmlAsync(query, ct).ConfigureAwait(false);
        if (!string.IsNullOrEmpty(web))
            return web;

        var wiki = await TryWikipediaAsync(query, ct).ConfigureAwait(false);
        if (!string.IsNullOrEmpty(wiki))
            return wiki;

        return null;
    }

    // Keep the first two keywords; nothing simpler worth retrying when already short.
    static string Simplify(string query)
    {
        if (string.IsNullOrWhiteSpace(query))
            return null;
        var parts = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
        if (parts.Length <= 2)
            return null;
        var simpler = parts[0] + " " + parts[1];
        return string.Equals(simpler, query, StringComparison.Ordinal) ? null : simpler;
    }

    // Korean question suffixes stripped from query tail; longest first to avoid partial matches.
    static readonly string[] s_KoreanTailFillers =
    {
        "에 대해서 자세히 설명해줘", "에 대해 자세히 설명해줘",
        "에 대해서 설명해줘", "에 대해 설명해줘", "에 대해서 알려줘", "에 대해 알려줘",
        "에 대해서 설명해", "에 대해 설명해", "에 대해서", "에 대해", "에 대한",
        "설명해줘", "설명 해줘", "설명해", "설명", "알려줘", "알려 줘",
        "검색해줘", "검색해", "찾아줘", "찾아 줘",
        "가 뭐야", "이 뭐야", "는 뭐야", "은 뭐야", "란 뭐야", "라는 게 뭐야",
        "가 뭔데", "이 뭔데", "뭐야", "뭔데", "가 누구야", "이 누구야", "누구야",
    };

    // English question prefixes stripped from query head; trailing space prevents touching compound words.
    static readonly string[] s_EnglishHeadFillers =
    {
        "tell me about ", "what is ", "what's ", "what are ", "who is ", "who's ",
        "explain ", "describe ", "search for ", "look up ", "about ",
    };

    static string CleanQuery(string query)
    {
        if (string.IsNullOrWhiteSpace(query)) return string.Empty;
        var q = query.Trim().TrimEnd('?', '.', '!', ',').Trim();

        var changed = true;
        while (changed)
        {
            changed = false;
            foreach (var f in s_KoreanTailFillers)
            {
                if (q.EndsWith(f, StringComparison.Ordinal))
                {
                    q = q.Substring(0, q.Length - f.Length).TrimEnd(' ', '?', '.', '!', ',').Trim();
                    changed = true;
                    break;
                }
            }
            foreach (var f in s_EnglishHeadFillers)
            {
                if (q.StartsWith(f, StringComparison.OrdinalIgnoreCase))
                {
                    q = q.Substring(f.Length).TrimStart().Trim();
                    changed = true;
                    break;
                }
            }
        }

        while (q.Contains("  ")) q = q.Replace("  ", " ");
        q = q.Trim();
        return string.IsNullOrWhiteSpace(q) ? query.Trim() : q;
    }

    static async Task<string> TryDuckDuckGoInstantAsync(string query, CancellationToken ct)
    {
        var url = DdgApiUrl + "?q=" + Uri.EscapeDataString(query) +
            "&format=json&no_html=1&skip_disambig=1&t=ondeviceagent";
        var json = await GetStringAsync(url, ct).ConfigureAwait(false);
        var root = JObject.Parse(json);

        var answer = (string)root["Answer"];
        if (!string.IsNullOrWhiteSpace(answer))
            return "Search result for \"" + query + "\": " + answer.Trim();

        var abstractText = (string)root["AbstractText"];
        if (!string.IsNullOrWhiteSpace(abstractText))
        {
            var source = (string)root["AbstractSource"];
            var tail = string.IsNullOrWhiteSpace(source) ? string.Empty : " (source: " + source.Trim() + ")";
            return "Search result for \"" + query + "\": " + Truncate(abstractText.Trim()) + tail;
        }

        var definition = (string)root["Definition"];
        if (!string.IsNullOrWhiteSpace(definition))
            return "Search result for \"" + query + "\": " + Truncate(definition.Trim());

        var related = root["RelatedTopics"] as JArray;
        if (related != null)
        {
            var sb = new StringBuilder();
            var count = 0;
            foreach (var item in related)
            {
                if (count >= 3) break;
                var text = (string)item["Text"];
                if (string.IsNullOrWhiteSpace(text)) continue;
                if (sb.Length > 0) sb.Append(" | ");
                sb.Append(text.Trim());
                count++;
            }
            if (sb.Length > 0)
                return "Search results for \"" + query + "\": " + Truncate(sb.ToString());
        }

        return null;
    }

    static async Task<string> TryDuckDuckGoHtmlAsync(string query, CancellationToken ct)
    {
        string html;
        try
        {
            var form = new FormUrlEncodedContent(new[]
            {
                new KeyValuePair<string, string>("q", query),
                new KeyValuePair<string, string>("kl", "wt-wt"),
            });
            using (var resp = await s_Http.PostAsync(DdgHtmlUrl, form, ct).ConfigureAwait(false))
            {
                if (!resp.IsSuccessStatusCode || resp.StatusCode == HttpStatusCode.Accepted)
                    return null;
                html = await resp.Content.ReadAsStringAsync().ConfigureAwait(false);
            }
        }
        catch (OperationCanceledException)
        {
            throw;
        }
        catch
        {
            return null;
        }

        var matches = Regex.Matches(html, "class=\"result__snippet\"[^>]*>(.*?)</a>", RegexOptions.Singleline);
        var sb = new StringBuilder();
        var count = 0;
        foreach (Match m in matches)
        {
            if (count >= 3) break;
            var text = StripHtml(m.Groups[1].Value);
            if (string.IsNullOrWhiteSpace(text)) continue;
            if (sb.Length > 0) sb.Append(" | ");
            sb.Append(text);
            count++;
        }

        if (sb.Length == 0)
            return null;

        return "Web results for \"" + query + "\": " + Truncate(sb.ToString());
    }

    static async Task<string> TryWikipediaAsync(string query, CancellationToken ct)
    {
        var primary = HasHangul(query) ? "ko" : "en";
        var hit = await WikiLookupAsync(primary, query, ct).ConfigureAwait(false);
        if (string.IsNullOrEmpty(hit) && primary != "en")
            hit = await WikiLookupAsync("en", query, ct).ConfigureAwait(false);
        return hit;
    }

    static async Task<string> WikiLookupAsync(string lang, string query, CancellationToken ct)
    {
        var apiBase = "https://" + lang + ".wikipedia.org/w/api.php";

        var searchUrl = apiBase + "?action=query&list=search&srlimit=3&format=json&srsearch=" +
            Uri.EscapeDataString(query);
        var searchJson = await GetStringAsync(searchUrl, ct).ConfigureAwait(false);
        var hits = JObject.Parse(searchJson)["query"]?["search"] as JArray;
        if (hits == null || hits.Count == 0)
            return null;

        var probe = Math.Min(2, hits.Count);
        for (var i = 0; i < probe; i++)
        {
            var title = (string)hits[i]["title"];
            if (string.IsNullOrWhiteSpace(title))
                continue;

            var extractUrl = apiBase + "?action=query&prop=extracts&exintro&explaintext&redirects=1&format=json&titles=" +
                Uri.EscapeDataString(title);
            var extractJson = await GetStringAsync(extractUrl, ct).ConfigureAwait(false);
            var pages = JObject.Parse(extractJson)["query"]?["pages"] as JObject;
            if (pages == null)
                continue;

            foreach (var page in pages.Properties())
            {
                var extract = (string)page.Value["extract"];
                if (!string.IsNullOrWhiteSpace(extract))
                    return "Wikipedia (" + title.Trim() + "): " + Truncate(extract.Trim());
            }
        }

        return null;
    }

    static bool HasHangul(string s)
    {
        if (string.IsNullOrEmpty(s)) return false;
        foreach (var ch in s)
        {
            if ((ch >= 0xAC00 && ch <= 0xD7A3) ||
                (ch >= 0x1100 && ch <= 0x11FF) ||
                (ch >= 0x3130 && ch <= 0x318F))
                return true;
        }
        return false;
    }

    static async Task<string> GetStringAsync(string url, CancellationToken ct)
    {
        using (var resp = await s_Http.GetAsync(url, ct).ConfigureAwait(false))
            {
                resp.EnsureSuccessStatusCode();
                return await resp.Content.ReadAsStringAsync().ConfigureAwait(false);
            }
        }

        static string StripHtml(string fragment)
        {
            if (string.IsNullOrEmpty(fragment)) return string.Empty;
            var noTags = Regex.Replace(fragment, "<[^>]+>", " ");
            var decoded = WebUtility.HtmlDecode(noTags);
            decoded = Regex.Replace(decoded, "\\s+", " ");
            return decoded.Trim();
        }

        static string Truncate(string text)
        {
            if (string.IsNullOrEmpty(text) || text.Length <= MaxSummaryChars)
                return text;
            return text.Substring(0, MaxSummaryChars) + "…";
        }

        string BuildApprovalRequiredMessage()
        {
            return "Approval required for tool " + Name + ". Choose an approval option in the modal to run this command.";
        }
    }
}